solid-jobs 0.1.3 → 0.2.0

data/lib/solid_jobs/heartbeat.rb

@@ -21,37 +21,39 @@ module SolidJobs
         "hostname" => Socket.gethostname,
         "pid" => ::Process.pid,
         "started_at" => @started_at,
-        "queues" => @config.queues,
+        "channels" => @config.channels,
         "labels" => [],
         "identity" => @identity,
       }
-      busy = @config.redis_pool.call("HLEN", "#{@identity}:work")
+      node_key = Keyspace.node(@identity)
+      work_key = Keyspace.node_work(@identity)
+      busy = @config.redis_pool.call("HLEN", work_key)
       @config.redis_pool.pipelined do |pipeline|
-        pipeline.call("INCRBY", "stat:processed", processed) if processed.positive?
-        pipeline.call("INCRBY", "stat:failed", failed) if failed.positive?
-        pipeline.call("SADD", "processes", @identity)
+        pipeline.call("INCRBY", Keyspace::PROCESSED, processed) if processed.positive?
+        pipeline.call("INCRBY", Keyspace::FAILED, failed) if failed.positive?
+        pipeline.call("SADD", Keyspace::NODES, @identity)
         pipeline.call(
           "HSET",
-          @identity,
+          node_key,
           "info", JSON.generate(info),
           "beat", now,
           "busy", busy,
           "quiet", quiet ? "true" : "false",
           "concurrency", @concurrency,
         )
-        pipeline.call("EXPIRE", @identity, TTL)
-        pipeline.call("DEL", "#{@identity}:work")
-        pipeline.call("HSET", "#{@identity}:work", *work.flatten(1)) unless work.empty?
-        pipeline.call("EXPIRE", "#{@identity}:work", TTL)
+        pipeline.call("EXPIRE", node_key, TTL)
+        pipeline.call("DEL", work_key)
+        pipeline.call("HSET", work_key, *work.flatten(1)) unless work.empty?
+        pipeline.call("EXPIRE", work_key, TTL)
       end
       now
     end
 
     def cleanup
       @config.redis_pool.pipelined do |pipeline|
-        pipeline.call("SREM", "processes", @identity)
-        pipeline.call("DEL", @identity)
-        pipeline.call("DEL", "#{@identity}:work")
+        pipeline.call("SREM", Keyspace::NODES, @identity)
+        pipeline.call("DEL", Keyspace.node(@identity))
+        pipeline.call("DEL", Keyspace.node_work(@identity))
       end
     ensure
       @config.close

data/lib/solid_jobs/integrity_check.rb

@@ -5,10 +5,10 @@ require "set"
 
 module SolidJobs
   class IntegrityCheck
-    ACTIVE_SETS = {
-      "schedule" => "SCHEDULED",
-      "retry" => "RETRY",
-      "dead" => "DEAD",
+    ACTIVE_COLLECTIONS = {
+      Keyspace::PLANNED => "PLANNED",
+      Keyspace::RETRIES => "RETRYING",
+      Keyspace::DISCARDED => "DISCARDED",
     }.freeze
 
     Report = Struct.new(
@@ -51,10 +51,10 @@ module SolidJobs
     end
 
     def call
-      read_queues
-      read_sorted_sets
-      reserved = read_reservations
-      validate_reservation_metadata(reserved)
+      read_channels
+      read_timed_collections
+      claimed = read_claimed
+      validate_claim_metadata(claimed)
       @acked.each { |job_id| add_state(job_id, "ACKED", "acknowledged") }
 
       observed = @states.keys.to_set
@@ -71,56 +71,57 @@ module SolidJobs
 
     private
 
-    def read_queues
-      scan("queue:*").each do |key|
+    def read_channels
+      scan("#{Keyspace::PREFIX}:channel:*").each do |key|
         Array(@redis.call("LRANGE", key, 0, -1)).each_with_index do |raw, index|
           add_payload(raw, "READY", "#{key}[#{index}]")
         end
       end
     end
 
-    def read_sorted_sets
-      ACTIVE_SETS.each do |key, state|
+    def read_timed_collections
+      ACTIVE_COLLECTIONS.each do |key, state|
         Array(@redis.call("ZRANGE", key, 0, -1)).each_with_index do |raw, index|
           add_payload(raw, state, "#{key}[#{index}]")
         end
       end
     end
 
-    def read_reservations
-      scan("*:reserved:*").to_h do |key|
+    def read_claimed
+      scan("#{Keyspace::PREFIX}:node:*:claimed:*").to_h do |key|
         records = Array(@redis.call("LRANGE", key, 0, -1)).each_with_index.map do |raw, index|
           payload = parse_payload(raw, "#{key}[#{index}]")
-          add_state(payload["jid"], "RESERVED", "#{key}[#{index}]") if payload
+          add_state(payload["id"], "CLAIMED", "#{key}[#{index}]") if payload
           payload
         end.compact
         [key, records]
       end
     end
 
-    def validate_reservation_metadata(reserved)
-      metadata = reservation_metadata
-      reserved.each do |key, payloads|
-        identity, worker_id = key.split(":reserved:", 2)
-        entry = metadata.delete([identity, worker_id])
+    def validate_claim_metadata(claimed)
+      metadata = claim_metadata
+      claimed.each do |key, payloads|
+        node_key, executor_id = key.split(":claimed:", 2)
+        identity = node_key.delete_prefix("#{Keyspace::PREFIX}:node:")
+        entry = metadata.delete([identity, executor_id])
         if payloads.empty?
-          @invalid_reservations << issue(key, "empty_reserved_list")
+          @invalid_reservations << issue(key, "empty_claimed_list")
         elsif payloads.length > 1
-          @invalid_reservations << issue(key, "multiple_payloads_for_worker")
+          @invalid_reservations << issue(key, "multiple_tasks_for_executor")
         end
         payload = payloads.first
         unless entry
-          @invalid_reservations << issue(key, "missing_metadata", job_id: payload&.[]("jid"))
+          @invalid_reservations << issue(key, "missing_metadata", job_id: payload&.[]("id"))
           next
         end
         next unless payload
 
         attributes = entry.fetch(:attributes)
         {
-          "job_id" => payload["jid"],
-          "process_id" => identity,
-          "worker_id" => worker_id,
-          "queue" => payload["queue"],
+          "task_id" => payload["id"],
+          "node_id" => identity,
+          "executor_id" => executor_id,
+          "channel" => payload["channel"],
         }.each do |field, expected|
           next if attributes[field].to_s == expected.to_s
 
@@ -138,14 +139,16 @@ module SolidJobs
           entry.fetch(:key),
           "metadata_without_payload",
           field: entry.fetch(:field),
-          job_id: entry.fetch(:attributes)["job_id"],
+          job_id: entry.fetch(:attributes)["task_id"],
         )
       end
     end
 
-    def reservation_metadata
-      scan("*:reservations").each_with_object({}) do |key, entries|
-        identity = key.delete_suffix(":reservations")
+    def claim_metadata
+      scan("#{Keyspace::PREFIX}:node:*:claims").each_with_object({}) do |key, entries|
+        identity = key
+          .delete_prefix("#{Keyspace::PREFIX}:node:")
+          .delete_suffix(":claims")
         hash_entries(key).each do |field, raw|
           attributes = parse_json(raw, "#{key}[#{field}]")
           next unless attributes
@@ -160,10 +163,10 @@ module SolidJobs
     end
 
     def dangling_attempts
-      hash_entries("solid-jobs:attempts").filter_map do |job_id, attempt|
+      hash_entries(Keyspace::ATTEMPTS).filter_map do |job_id, attempt|
         next if @states[job_id]&.any? { |location| location.fetch(:state) != "ACKED" }
 
-        {job_id: job_id, attempt: Integer(attempt), index: "solid-jobs:attempts"}
+        {job_id: job_id, attempt: Integer(attempt), index: Keyspace::ATTEMPTS}
       end
     end
 
@@ -177,15 +180,15 @@ module SolidJobs
 
     def add_payload(raw, state, location)
       payload = parse_payload(raw, location)
-      add_state(payload["jid"], state, location) if payload
+      add_state(payload["id"], state, location) if payload
     end
 
     def parse_payload(raw, location)
       payload = parse_json(raw, location)
       return unless payload
-      return payload if payload["jid"]
+      return payload if payload["id"]
 
-      @malformed_payloads << {location: location, error: "missing jid", raw: raw}
+      @malformed_payloads << {location: location, error: "missing id", raw: raw}
       nil
     end
 

data/lib/solid_jobs/interceptor_registry.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  class InterceptorRegistry
+    Definition = Data.define(:type, :arguments)
+
+    include Enumerable
+
+    def initialize(definitions = [])
+      @definitions = definitions.map do |definition|
+        Definition.new(definition.type, definition.arguments.dup.freeze)
+      end
+    end
+
+    def each(&block)
+      @definitions.each(&block)
+    end
+
+    def use(type, *arguments)
+      discard(type)
+      @definitions = (@definitions + [Definition.new(type, arguments.freeze)]).freeze
+      self
+    end
+
+    def discard(type)
+      @definitions = @definitions.reject { |definition| definition.type == type }.freeze
+      self
+    end
+
+    def empty?
+      @definitions.empty?
+    end
+
+    def call(context, &operation)
+      continuation = operation
+      @definitions.reverse_each do |definition|
+        interceptor = definition.type.new(*definition.arguments)
+        downstream = continuation
+        continuation = -> { interceptor.around(context, &downstream) }
+      end
+      continuation.call
+    end
+
+    def export
+      Utilities.shareable_copy(
+        @definitions.map { |definition| [definition.type, definition.arguments] },
+      )
+    end
+
+    def import(records)
+      @definitions = records.map do |type, arguments|
+        Definition.new(type, arguments.freeze)
+      end.freeze
+      self
+    end
+  end
+end

data/lib/solid_jobs/keyspace.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  module Keyspace
+    PREFIX = "solid_jobs".freeze
+    CHANNELS = "#{PREFIX}:channels".freeze
+    PAUSED_CHANNELS = "#{PREFIX}:paused_channels".freeze
+    PLANNED = "#{PREFIX}:planned".freeze
+    RETRIES = "#{PREFIX}:retries".freeze
+    DISCARDED = "#{PREFIX}:discarded".freeze
+    NODES = "#{PREFIX}:nodes".freeze
+    ATTEMPTS = "#{PREFIX}:attempts".freeze
+    PROCESSED = "#{PREFIX}:metrics:processed".freeze
+    FAILED = "#{PREFIX}:metrics:failed".freeze
+
+    module_function
+
+    def channel(name)
+      "#{PREFIX}:channel:#{name}"
+    end
+
+    def node(identity)
+      "#{PREFIX}:node:#{identity}"
+    end
+
+    def node_work(identity)
+      "#{node(identity)}:work"
+    end
+
+    def node_signals(identity)
+      "#{node(identity)}:signals"
+    end
+
+    def claims(identity)
+      "#{node(identity)}:claims"
+    end
+
+    def claimed(identity, executor_id)
+      "#{node(identity)}:claimed:#{executor_id}"
+    end
+  end
+end

data/lib/solid_jobs/processor.rb

@@ -6,8 +6,8 @@ module SolidJobs
   class Processor
     def initialize(config, identity: nil, processor_id: nil, heartbeat: nil)
       @config = config
-      @fetcher = Fetch.new(config, identity: identity, processor_id: processor_id)
-      @worker = Worker.new(redis_config: config.redis_config, config: config)
+      @claims = Claim.new(config, identity: identity, processor_id: processor_id)
+      @executor = Executor.new(redis_config: config.redis_config, config: config)
       @identity = identity
       @processor_id = processor_id
       @heartbeat = heartbeat
@@ -29,12 +29,12 @@ module SolidJobs
           next
         end
 
-        unit = retrieve
-        next unless unit
+        claim = retrieve
+        next unless claim
 
         receive_commands(control)
         if @stopping || @quiet
-          requeue(unit)
+          requeue(claim)
           break if @stopping
 
           next
@@ -42,19 +42,19 @@ module SolidJobs
 
         begin
           @state_mutex.synchronize { @busy = true }
-          payload = unit.job
-          register_work(unit, payload)
+          envelope = claim.envelope
+          register_work(claim, envelope)
           begin
-            @worker.perform(payload)
+            @executor.execute(envelope)
           rescue Shutdown
-            requeue(unit)
+            requeue(claim)
             break
           rescue StandardError => error
-            RetryService.call(config: @config, payload: payload, error: error)
-            acknowledge(unit)
+            FailurePolicy.call(config: @config, payload: envelope, error: error)
+            complete(claim)
             failed += 1
           else
-            processed += 1 if acknowledge(unit)
+            processed += 1 if complete(claim)
           end
         ensure
           @state_mutex.synchronize { @busy = false }
@@ -78,50 +78,50 @@ module SolidJobs
 
     private
 
-    def acknowledge(unit)
-      acknowledged = unit.acknowledge
-      unless acknowledged
+    def complete(claim)
+      completed = claim.complete
+      unless completed
         @config.logger.warn(
-          "Reservation fencing rejected stale ACK: #{unit.reservation_id}",
+          "Claim fencing rejected stale completion: #{claim.claim_token}",
         )
       end
-      acknowledged
+      completed
     rescue SolidRedis::ConnectionError => error
       @fetcher.connection_failed!
       @config.logger.warn(
-        "Redis ACK failed; reserved job will be replayed: #{error.class}: #{error.message}",
+        "Redis completion failed; claimed task will be replayed: #{error.class}: #{error.message}",
       )
       false
     end
 
-    def requeue(unit)
-      requeued = unit.requeue
+    def requeue(claim)
+      requeued = claim.requeue
       unless requeued
         @config.logger.warn(
-          "Reservation fencing rejected stale requeue: #{unit.reservation_id}",
+          "Claim fencing rejected stale requeue: #{claim.claim_token}",
         )
       end
       requeued
     end
 
     def retrieve
-      @fetcher.retrieve
+      @claims.next
     rescue SolidRedis::ConnectionError => error
-      @fetcher.connection_failed!
-      @config.logger.warn("Redis fetch failed, retrying: #{error.class}: #{error.message}")
+      @claims.connection_failed!
+      @config.logger.warn("Redis claim failed, retrying: #{error.class}: #{error.message}")
       sleep 0.1
       nil
     end
 
-    def register_work(unit, payload)
+    def register_work(claim, envelope)
       return unless @identity
 
       work = JSON.generate(
-        "queue" => unit.queue,
-        "payload" => payload,
-        "run_at" => Time.now.to_f,
-        "reservation_id" => unit.reservation_id,
-        "attempt" => unit.attempt,
+        "channel" => claim.channel,
+        "envelope" => envelope,
+        "started_at" => Time.now.to_f,
+        "claim_token" => claim.claim_token,
+        "attempt" => claim.attempt,
       )
       send_heartbeat(:work, @processor_id, work)
     end

data/lib/solid_jobs/publisher.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+
+module SolidJobs
+  class Publisher
+    DEFAULT_CHUNK_SIZE = 1_000
+    PLANNED_CHUNK_SIZE = 100
+    Publication = Data.define(:task, :envelope, :redis_pool)
+
+    def self.publish(envelope)
+      new.publish(envelope)
+    end
+
+    def self.publish_many(envelope)
+      new.publish_many(envelope)
+    end
+
+    def initialize(config: nil, configuration: nil)
+      selected = config || configuration || SolidJobs.config
+      @config = selected.is_a?(SolidRedis::Config) ? Config.new(redis: selected) : selected
+    end
+
+    def publish(envelope)
+      normalized = normalize(envelope)
+      normalized = intercept(normalized)
+      return unless normalized
+
+      validate_arguments!(normalized.fetch("arguments"))
+      persist([normalized])
+      normalized.fetch("id")
+    end
+
+    def publish_many(envelope)
+      raise InvalidJobError, "Task envelope must be a Hash" unless envelope.is_a?(Hash)
+
+      source = Utilities.stringify_hash_keys(envelope)
+      argument_sets = source.delete("arguments")
+      unless argument_sets.respond_to?(:each)
+        raise InvalidJobError, "Task 'arguments' must be an Array or Enumerable"
+      end
+
+      run_times = source.delete("run_at")
+      unless run_times.nil? || run_times.is_a?(Numeric) || run_times.is_a?(Array)
+        raise InvalidJobError, "Task 'run_at' must be Numeric or an Array"
+      end
+      if run_times.is_a?(Array) && argument_sets.respond_to?(:size) && run_times.size != argument_sets.size
+        raise InvalidJobError, "Task 'run_at' Array must match the number of argument sets"
+      end
+
+      chunk_size = Integer(
+        source.delete("chunk_size") || (run_times ? PLANNED_CHUNK_SIZE : DEFAULT_CHUNK_SIZE),
+      )
+      raise ArgumentError, "chunk_size must be positive" unless chunk_size.positive?
+
+      explicit_id = source.delete("id")
+      if explicit_id && argument_sets.respond_to?(:size) && argument_sets.size > 1
+        raise InvalidJobError, "An explicit id is only valid for a single task"
+      end
+
+      template = normalize(source.merge("arguments" => []))
+      template.delete("id")
+      ids = []
+      argument_sets.each_slice(chunk_size).with_index do |chunk, chunk_index|
+        envelopes = chunk.filter_map.with_index do |arguments, index|
+          raise InvalidJobError, "Each task's arguments must be an Array" unless arguments.is_a?(Array)
+
+          item = template.merge(
+            "arguments" => arguments,
+            "id" => explicit_id || SecureRandom.uuid,
+          )
+          item["run_at"] = run_times.is_a?(Array) ? run_times[chunk_index * chunk_size + index] : run_times if run_times
+          intercepted = intercept(item)
+          if intercepted
+            validate_arguments!(intercepted.fetch("arguments"))
+            ids << intercepted.fetch("id")
+          else
+            ids << nil
+          end
+          intercepted
+        end
+        persist(envelopes)
+      end
+      ids
+    end
+
+    def persist(envelopes)
+      return true if envelopes.empty?
+
+      if defined?(Testing)
+        case Testing.mode
+        when :capture
+          envelopes.each do |envelope|
+            Testing.captured_for(Utilities.constantize(envelope.fetch("task"))) << envelope
+          end
+          return true
+        when :execute
+          envelopes.each { |envelope| Testing.execute(envelope) }
+          return true
+        end
+      end
+
+      @config.redis_pool.pipelined do |pipeline|
+        planned, ready = envelopes.partition { |envelope| envelope["run_at"] }
+        planned.each do |envelope|
+          score = envelope.delete("run_at")
+          pipeline.call("ZADD", Keyspace::PLANNED, score, JSON.generate(envelope))
+        end
+        persist_ready(pipeline, ready)
+      end
+      true
+    end
+
+    private
+
+    def normalize(envelope)
+      raise InvalidJobError, "Task envelope must be a Hash" unless envelope.is_a?(Hash)
+
+      item = @config.default_task_options.merge(Utilities.stringify_hash_keys(envelope))
+      task = item["task"]
+      arguments = item["arguments"]
+      unless task.is_a?(Class) || (task.is_a?(String) && !task.empty?)
+        raise InvalidJobError, "Task envelope requires a non-empty task"
+      end
+      raise InvalidJobError, "Task arguments must be an Array" unless arguments.is_a?(Array)
+
+      item["task"] = task.name if task.is_a?(Class)
+      item["channel"] = String(item.fetch("channel"))
+      item["id"] ||= SecureRandom.uuid
+      item["created_ms"] ||= Utilities.realtime_milliseconds
+      item
+    end
+
+    def intercept(envelope)
+      context = Publication.new(envelope.fetch("task"), envelope, @config.redis_pool)
+      @config.publish_interceptors.call(context) { envelope }
+    end
+
+    def persist_ready(pipeline, envelopes)
+      envelopes.group_by { |envelope| envelope.fetch("channel") }.each do |channel, group|
+        pipeline.call("SADD", Keyspace::CHANNELS, channel)
+        queued_ms = Utilities.realtime_milliseconds
+        encoded = group.map do |envelope|
+          envelope["queued_ms"] = queued_ms
+          JSON.generate(envelope)
+        end
+        pipeline.call("LPUSH", Keyspace.channel(channel), *encoded)
+      end
+    end
+
+    def validate_arguments!(arguments)
+      invalid = invalid_json_value(arguments)
+      return unless invalid
+
+      message = "Task arguments must contain JSON-native values with string hash keys; found #{invalid.inspect}"
+      case @config.on_complex_arguments
+      when :raise then raise InvalidArgumentError, message
+      when :warn then @config.logger.warn(message)
+      end
+    end
+
+    def invalid_json_value(value)
+      case value
+      when nil, true, false, String, Integer, Float
+        nil
+      when Array
+        value.each do |nested|
+          invalid = invalid_json_value(nested)
+          return invalid if invalid
+        end
+        nil
+      when Hash
+        value.each do |key, nested|
+          return key unless key.is_a?(String)
+
+          invalid = invalid_json_value(nested)
+          return invalid if invalid
+        end
+        nil
+      else
+        value
+      end
+    end
+  end
+end

data/lib/solid_jobs/recovery.rb

@@ -5,12 +5,12 @@ require "socket"
 
 module SolidJobs
   class Recovery < Service::Base
-    RESERVED_PATTERN = "*:reserved:*"
+    CLAIMED_PATTERN = "#{Keyspace::PREFIX}:node:*:claimed:*".freeze
     RESTORE_ONE = <<~LUA.freeze
       local payload = redis.call("rpop", KEYS[1])
       if payload then
-        local job = cjson.decode(payload)
-        redis.call("rpush", "queue:" .. job["queue"], payload)
+        local envelope = cjson.decode(payload)
+        redis.call("rpush", "#{Keyspace::PREFIX}:channel:" .. envelope["channel"], payload)
         return payload
       end
     LUA
@@ -18,14 +18,15 @@ module SolidJobs
     def call
       recovered = 0
       scan_keys.each do |key|
-        identity = key.split(":reserved:", 2).first
+        node_key = key.split(":claimed:", 2).first
+        identity = node_key.delete_prefix("#{Keyspace::PREFIX}:node:")
         next if alive?(identity)
 
         while @config.redis_pool.call("EVAL", RESTORE_ONE, 1, key)
           recovered += 1
         end
         @config.redis_pool.call("DEL", key)
-        @config.redis_pool.call("DEL", "#{identity}:reservations")
+        @config.redis_pool.call("DEL", Keyspace.claims(identity))
       end
       recovered
     end
@@ -36,7 +37,7 @@ module SolidJobs
       hostname, pid = identity.split(":", 3)
       return false if hostname == Socket.gethostname && !process_alive?(Integer(pid))
 
-      beat = @config.redis_pool.call("HGET", identity, "beat")
+      beat = @config.redis_pool.call("HGET", Keyspace.node(identity), "beat")
       beat && Time.now.to_f - Float(beat) < Heartbeat::TTL
     end
 
@@ -54,7 +55,7 @@ module SolidJobs
       keys = []
       loop do
         cursor, found = @config.redis_pool.call(
-          "SCAN", cursor, "MATCH", RESERVED_PATTERN, "COUNT", 100,
+          "SCAN", cursor, "MATCH", CLAIMED_PATTERN, "COUNT", 100,
         )
         keys.concat(found)
         break if cursor == "0"