solid-jobs 0.1.3 → 0.4.0

data/lib/solid_jobs/failure_policy.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SolidJobs
+  class FailurePolicy < Service::Base
+
+    def call
+      payload = @payload.dup
+      now = Time.now
+      now_ms = Utilities.realtime_milliseconds
+      failure_count = Integer(payload.fetch("failure_count", 0)) + 1
+      payload["exception_message"] = @error.message.to_s
+      payload["exception_type"] = @error.class.name
+      payload["first_failed_ms"] ||= now_ms
+      payload["last_failed_ms"] = now_ms
+      payload["failure_count"] = failure_count
+      payload["channel"] = payload["retry_channel"] if payload["retry_channel"]
+
+      if retry_allowed?(payload, failure_count, now)
+        decision = retry_decision(payload, failure_count)
+        return nil if decision == :drop
+        return archive(payload, now.to_f) if decision == :archive
+
+        next_attempt = now.to_f + (decision || retry_delay(failure_count))
+        @config.redis_pool.call("ZADD", Keyspace::RETRIES, next_attempt, JSON.generate(payload))
+        append_message("retry scheduled")
+        next_attempt
+      else
+        invoke_final_failure(payload)
+        archive(payload, now.to_f)
+        append_message("task discarded")
+        nil
+      end
+    end
+
+    private
+
+    def retry_allowed?(payload, failure_count, now)
+      if payload["retry_within"]
+        first_failure = Float(payload["first_failed_ms"]) / 1_000
+        now.to_f - first_failure < Float(payload["retry_within"])
+      else
+        maximum = Integer(payload.fetch("max_failures", 0))
+        failure_count <= maximum
+      end
+    end
+
+    def retry_delay(failure_count)
+      ceiling = [
+        Float(@config.retry_base_delay) * (2**(failure_count - 1)),
+        Float(@config.retry_max_delay),
+      ].min
+      (ceiling / 2) + (rand * ceiling / 2)
+    end
+
+    def retry_decision(payload, failure_count)
+      task = resolve_task(payload)
+      return unless task.respond_to?(:retry_delay_callback)
+
+      value = task.retry_delay_callback(failure_count, @error, payload)
+      return value if value.is_a?(Numeric) || %i[drop archive].include?(value)
+
+      nil
+    end
+
+    def invoke_final_failure(payload)
+      task = resolve_task(payload)
+      return unless task.respond_to?(:final_failure_callback)
+
+      task.final_failure_callback(payload, @error)
+    end
+
+    def resolve_task(payload)
+      Utilities.constantize(payload.fetch("task"))
+    rescue NameError
+      nil
+    end
+
+    def archive(payload, timestamp)
+      @config.redis_pool.pipelined do |pipeline|
+        pipeline.call("ZADD", Keyspace::DISCARDED, timestamp, JSON.generate(payload))
+        pipeline.call(
+          "ZREMRANGEBYSCORE",
+          Keyspace::DISCARDED,
+          "-inf",
+          timestamp - @config.discarded_retention,
+        )
+        pipeline.call(
+          "ZREMRANGEBYRANK",
+          Keyspace::DISCARDED,
+          0,
+          -(@config.discarded_limit + 1),
+        )
+      end
+    end
+  end
+end

data/lib/solid_jobs/heartbeat.rb

@@ -21,37 +21,39 @@ module SolidJobs
         "hostname" => Socket.gethostname,
         "pid" => ::Process.pid,
         "started_at" => @started_at,
-        "queues" => @config.queues,
+        "channels" => @config.channels,
         "labels" => [],
         "identity" => @identity,
       }
-      busy = @config.redis_pool.call("HLEN", "#{@identity}:work")
+      node_key = Keyspace.node(@identity)
+      work_key = Keyspace.node_work(@identity)
+      busy = @config.redis_pool.call("HLEN", work_key)
       @config.redis_pool.pipelined do |pipeline|
-        pipeline.call("INCRBY", "stat:processed", processed) if processed.positive?
-        pipeline.call("INCRBY", "stat:failed", failed) if failed.positive?
-        pipeline.call("SADD", "processes", @identity)
+        pipeline.call("INCRBY", Keyspace::PROCESSED, processed) if processed.positive?
+        pipeline.call("INCRBY", Keyspace::FAILED, failed) if failed.positive?
+        pipeline.call("SADD", Keyspace::NODES, @identity)
         pipeline.call(
           "HSET",
-          @identity,
+          node_key,
           "info", JSON.generate(info),
           "beat", now,
           "busy", busy,
           "quiet", quiet ? "true" : "false",
           "concurrency", @concurrency,
         )
-        pipeline.call("EXPIRE", @identity, TTL)
-        pipeline.call("DEL", "#{@identity}:work")
-        pipeline.call("HSET", "#{@identity}:work", *work.flatten(1)) unless work.empty?
-        pipeline.call("EXPIRE", "#{@identity}:work", TTL)
+        pipeline.call("EXPIRE", node_key, TTL)
+        pipeline.call("DEL", work_key)
+        pipeline.call("HSET", work_key, *work.flatten(1)) unless work.empty?
+        pipeline.call("EXPIRE", work_key, TTL)
       end
       now
     end
 
     def cleanup
       @config.redis_pool.pipelined do |pipeline|
-        pipeline.call("SREM", "processes", @identity)
-        pipeline.call("DEL", @identity)
-        pipeline.call("DEL", "#{@identity}:work")
+        pipeline.call("SREM", Keyspace::NODES, @identity)
+        pipeline.call("DEL", Keyspace.node(@identity))
+        pipeline.call("DEL", Keyspace.node_work(@identity))
       end
     ensure
       @config.close

data/lib/solid_jobs/integrity_check.rb

@@ -5,10 +5,10 @@ require "set"
 
 module SolidJobs
   class IntegrityCheck
-    ACTIVE_SETS = {
-      "schedule" => "SCHEDULED",
-      "retry" => "RETRY",
-      "dead" => "DEAD",
+    ACTIVE_COLLECTIONS = {
+      Keyspace::PLANNED => "PLANNED",
+      Keyspace::RETRIES => "RETRYING",
+      Keyspace::DISCARDED => "DISCARDED",
     }.freeze
 
     Report = Struct.new(
@@ -51,10 +51,10 @@ module SolidJobs
     end
 
     def call
-      read_queues
-      read_sorted_sets
-      reserved = read_reservations
-      validate_reservation_metadata(reserved)
+      read_channels
+      read_timed_collections
+      claimed = read_claimed
+      validate_claim_metadata(claimed)
       @acked.each { |job_id| add_state(job_id, "ACKED", "acknowledged") }
 
       observed = @states.keys.to_set
@@ -71,56 +71,57 @@ module SolidJobs
 
     private
 
-    def read_queues
-      scan("queue:*").each do |key|
+    def read_channels
+      scan("#{Keyspace::PREFIX}:channel:*").each do |key|
         Array(@redis.call("LRANGE", key, 0, -1)).each_with_index do |raw, index|
           add_payload(raw, "READY", "#{key}[#{index}]")
         end
       end
     end
 
-    def read_sorted_sets
-      ACTIVE_SETS.each do |key, state|
+    def read_timed_collections
+      ACTIVE_COLLECTIONS.each do |key, state|
         Array(@redis.call("ZRANGE", key, 0, -1)).each_with_index do |raw, index|
           add_payload(raw, state, "#{key}[#{index}]")
         end
       end
     end
 
-    def read_reservations
-      scan("*:reserved:*").to_h do |key|
+    def read_claimed
+      scan("#{Keyspace::PREFIX}:node:*:claimed:*").to_h do |key|
         records = Array(@redis.call("LRANGE", key, 0, -1)).each_with_index.map do |raw, index|
           payload = parse_payload(raw, "#{key}[#{index}]")
-          add_state(payload["jid"], "RESERVED", "#{key}[#{index}]") if payload
+          add_state(payload["id"], "CLAIMED", "#{key}[#{index}]") if payload
           payload
         end.compact
         [key, records]
       end
     end
 
-    def validate_reservation_metadata(reserved)
-      metadata = reservation_metadata
-      reserved.each do |key, payloads|
-        identity, worker_id = key.split(":reserved:", 2)
-        entry = metadata.delete([identity, worker_id])
+    def validate_claim_metadata(claimed)
+      metadata = claim_metadata
+      claimed.each do |key, payloads|
+        node_key, executor_id = key.split(":claimed:", 2)
+        identity = node_key.delete_prefix("#{Keyspace::PREFIX}:node:")
+        entry = metadata.delete([identity, executor_id])
         if payloads.empty?
-          @invalid_reservations << issue(key, "empty_reserved_list")
+          @invalid_reservations << issue(key, "empty_claimed_list")
         elsif payloads.length > 1
-          @invalid_reservations << issue(key, "multiple_payloads_for_worker")
+          @invalid_reservations << issue(key, "multiple_tasks_for_executor")
         end
         payload = payloads.first
         unless entry
-          @invalid_reservations << issue(key, "missing_metadata", job_id: payload&.[]("jid"))
+          @invalid_reservations << issue(key, "missing_metadata", job_id: payload&.[]("id"))
           next
         end
         next unless payload
 
         attributes = entry.fetch(:attributes)
         {
-          "job_id" => payload["jid"],
-          "process_id" => identity,
-          "worker_id" => worker_id,
-          "queue" => payload["queue"],
+          "task_id" => payload["id"],
+          "node_id" => identity,
+          "executor_id" => executor_id,
+          "channel" => payload["channel"],
         }.each do |field, expected|
           next if attributes[field].to_s == expected.to_s
 
@@ -138,14 +139,16 @@ module SolidJobs
           entry.fetch(:key),
           "metadata_without_payload",
           field: entry.fetch(:field),
-          job_id: entry.fetch(:attributes)["job_id"],
+          job_id: entry.fetch(:attributes)["task_id"],
         )
       end
     end
 
-    def reservation_metadata
-      scan("*:reservations").each_with_object({}) do |key, entries|
-        identity = key.delete_suffix(":reservations")
+    def claim_metadata
+      scan("#{Keyspace::PREFIX}:node:*:claims").each_with_object({}) do |key, entries|
+        identity = key
+          .delete_prefix("#{Keyspace::PREFIX}:node:")
+          .delete_suffix(":claims")
         hash_entries(key).each do |field, raw|
           attributes = parse_json(raw, "#{key}[#{field}]")
           next unless attributes
@@ -160,10 +163,10 @@ module SolidJobs
     end
 
     def dangling_attempts
-      hash_entries("solid-jobs:attempts").filter_map do |job_id, attempt|
+      hash_entries(Keyspace::ATTEMPTS).filter_map do |job_id, attempt|
         next if @states[job_id]&.any? { |location| location.fetch(:state) != "ACKED" }
 
-        {job_id: job_id, attempt: Integer(attempt), index: "solid-jobs:attempts"}
+        {job_id: job_id, attempt: Integer(attempt), index: Keyspace::ATTEMPTS}
       end
     end
 
@@ -177,15 +180,15 @@ module SolidJobs
 
     def add_payload(raw, state, location)
       payload = parse_payload(raw, location)
-      add_state(payload["jid"], state, location) if payload
+      add_state(payload["id"], state, location) if payload
     end
 
     def parse_payload(raw, location)
       payload = parse_json(raw, location)
       return unless payload
-      return payload if payload["jid"]
+      return payload if payload["id"]
 
-      @malformed_payloads << {location: location, error: "missing jid", raw: raw}
+      @malformed_payloads << {location: location, error: "missing id", raw: raw}
       nil
     end
 

data/lib/solid_jobs/interceptor_registry.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  class InterceptorRegistry
+    Definition = Data.define(:type, :arguments)
+
+    include Enumerable
+
+    def initialize(definitions = [])
+      @definitions = definitions.map do |definition|
+        Definition.new(definition.type, definition.arguments.dup.freeze)
+      end
+    end
+
+    def each(&block)
+      @definitions.each(&block)
+    end
+
+    def use(type, *arguments)
+      discard(type)
+      @definitions = (@definitions + [Definition.new(type, arguments.freeze)]).freeze
+      self
+    end
+
+    def discard(type)
+      @definitions = @definitions.reject { |definition| definition.type == type }.freeze
+      self
+    end
+
+    def empty?
+      @definitions.empty?
+    end
+
+    def call(context, &operation)
+      continuation = operation
+      @definitions.reverse_each do |definition|
+        interceptor = definition.type.new(*definition.arguments)
+        downstream = continuation
+        continuation = -> { interceptor.around(context, &downstream) }
+      end
+      continuation.call
+    end
+
+    def export
+      Utilities.shareable_copy(
+        @definitions.map { |definition| [definition.type, definition.arguments] },
+      )
+    end
+
+    def import(records)
+      @definitions = records.map do |type, arguments|
+        Definition.new(type, arguments.freeze)
+      end.freeze
+      self
+    end
+  end
+end

data/lib/solid_jobs/keyspace.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  module Keyspace
+    PREFIX = "solid_jobs".freeze
+    CHANNELS = "#{PREFIX}:channels".freeze
+    PAUSED_CHANNELS = "#{PREFIX}:paused_channels".freeze
+    PLANNED = "#{PREFIX}:planned".freeze
+    RETRIES = "#{PREFIX}:retries".freeze
+    DISCARDED = "#{PREFIX}:discarded".freeze
+    NODES = "#{PREFIX}:nodes".freeze
+    ATTEMPTS = "#{PREFIX}:attempts".freeze
+    PROCESSED = "#{PREFIX}:metrics:processed".freeze
+    FAILED = "#{PREFIX}:metrics:failed".freeze
+
+    module_function
+
+    def channel(name)
+      "#{PREFIX}:channel:#{name}"
+    end
+
+    def node(identity)
+      "#{PREFIX}:node:#{identity}"
+    end
+
+    def node_work(identity)
+      "#{node(identity)}:work"
+    end
+
+    def node_signals(identity)
+      "#{node(identity)}:signals"
+    end
+
+    def claims(identity)
+      "#{node(identity)}:claims"
+    end
+
+    def claimed(identity, executor_id)
+      "#{node(identity)}:claimed:#{executor_id}"
+    end
+  end
+end

data/lib/solid_jobs/{testing.rb → lab.rb}

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
+require "securerandom"
+
 module SolidJobs
-  module Testing
-    MODES = %i[disable fake inline].freeze
-    STORAGE_KEY = :solid_jobs_testing_jobs
+  module Lab
+    MODES = %i[disable capture execute].freeze
+    STORAGE_KEY = :solid_jobs_testing_captured
     MODE_KEY = :solid_jobs_testing_mode
 
     module_function
@@ -29,14 +31,14 @@ module SolidJobs
       end
     end
 
-    def fake!
-      testing!(:fake) { yield } if block_given?
-      testing!(:fake) unless block_given?
+    def capture!
+      testing!(:capture) { yield } if block_given?
+      testing!(:capture) unless block_given?
     end
 
-    def inline!
-      testing!(:inline) { yield } if block_given?
-      testing!(:inline) unless block_given?
+    def execute!
+      testing!(:execute) { yield } if block_given?
+      testing!(:execute) unless block_given?
     end
 
     def disable!
@@ -44,22 +46,20 @@ module SolidJobs
       testing!(:disable) unless block_given?
     end
 
-    def jobs_for(job_class)
-      storage[job_class.name] ||= []
+    def captured_for(task)
+      storage[task.name] ||= []
     end
 
     def clear_all
       storage.clear
     end
 
-    def execute_inline(payload)
-      job_class = payload["class"]
-      job_class = Utilities.constantize(job_class) if job_class.is_a?(String)
-      instance = job_class.new
-      instance.jid = payload["jid"] if instance.respond_to?(:jid=)
-      SolidJobs.config.server_middleware.invoke(instance, payload, payload.fetch("queue", "default")) do
-        instance.perform(*payload.fetch("args"))
-      end
+    def execute(envelope)
+      normalized = Utilities.stringify_hash_keys(envelope)
+      normalized["task"] = normalized.fetch("task").name if normalized["task"].is_a?(Class)
+      normalized["channel"] ||= "default"
+      normalized["id"] ||= SecureRandom.uuid
+      Executor.new(redis_config: SolidJobs.config.redis_config).execute(normalized)
     end
 
     def storage

data/lib/solid_jobs/publisher.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+
+module SolidJobs
+  class Publisher
+    DEFAULT_CHUNK_SIZE = 1_000
+    PLANNED_CHUNK_SIZE = 100
+    Publication = Data.define(:task, :envelope, :redis_pool)
+
+    def self.publish(envelope)
+      new.publish(envelope)
+    end
+
+    def self.publish_many(envelope)
+      new.publish_many(envelope)
+    end
+
+    def initialize(config: nil, configuration: nil)
+      selected = config || configuration || SolidJobs.config
+      @config = selected.is_a?(SolidRedis::Config) ? Blueprint.new(redis: selected) : selected
+    end
+
+    def publish(envelope)
+      normalized = normalize(envelope)
+      normalized = intercept(normalized)
+      return unless normalized
+
+      validate_arguments!(normalized.fetch("arguments"))
+      persist([normalized])
+      normalized.fetch("id")
+    end
+
+    def publish_many(envelope)
+      raise InvalidJobError, "Task envelope must be a Hash" unless envelope.is_a?(Hash)
+
+      source = Utilities.stringify_hash_keys(envelope)
+      argument_sets = source.delete("arguments")
+      unless argument_sets.respond_to?(:each)
+        raise InvalidJobError, "Task 'arguments' must be an Array or Enumerable"
+      end
+
+      run_times = source.delete("run_at")
+      unless run_times.nil? || run_times.is_a?(Numeric) || run_times.is_a?(Array)
+        raise InvalidJobError, "Task 'run_at' must be Numeric or an Array"
+      end
+      if run_times.is_a?(Array) && argument_sets.respond_to?(:size) && run_times.size != argument_sets.size
+        raise InvalidJobError, "Task 'run_at' Array must match the number of argument sets"
+      end
+
+      chunk_size = Integer(
+        source.delete("chunk_size") || (run_times ? PLANNED_CHUNK_SIZE : DEFAULT_CHUNK_SIZE),
+      )
+      raise ArgumentError, "chunk_size must be positive" unless chunk_size.positive?
+
+      explicit_id = source.delete("id")
+      if explicit_id && argument_sets.respond_to?(:size) && argument_sets.size > 1
+        raise InvalidJobError, "An explicit id is only valid for a single task"
+      end
+
+      template = normalize(source.merge("arguments" => []))
+      template.delete("id")
+      ids = []
+      argument_sets.each_slice(chunk_size).with_index do |chunk, chunk_index|
+        envelopes = chunk.filter_map.with_index do |arguments, index|
+          raise InvalidJobError, "Each task's arguments must be an Array" unless arguments.is_a?(Array)
+
+          item = template.merge(
+            "arguments" => arguments,
+            "id" => explicit_id || SecureRandom.uuid,
+          )
+          item["run_at"] = run_times.is_a?(Array) ? run_times[chunk_index * chunk_size + index] : run_times if run_times
+          intercepted = intercept(item)
+          if intercepted
+            validate_arguments!(intercepted.fetch("arguments"))
+            ids << intercepted.fetch("id")
+          else
+            ids << nil
+          end
+          intercepted
+        end
+        persist(envelopes)
+      end
+      ids
+    end
+
+    def persist(envelopes)
+      return true if envelopes.empty?
+
+      if defined?(Lab)
+        case Lab.mode
+        when :capture
+          envelopes.each do |envelope|
+            Lab.captured_for(Utilities.constantize(envelope.fetch("task"))) << envelope
+          end
+          return true
+        when :execute
+          envelopes.each { |envelope| Lab.execute(envelope) }
+          return true
+        end
+      end
+
+      @config.redis_pool.pipelined do |pipeline|
+        planned, ready = envelopes.partition { |envelope| envelope["run_at"] }
+        planned.each do |envelope|
+          score = envelope.delete("run_at")
+          pipeline.call("ZADD", Keyspace::PLANNED, score, JSON.generate(envelope))
+        end
+        persist_ready(pipeline, ready)
+      end
+      true
+    end
+
+    private
+
+    def normalize(envelope)
+      raise InvalidJobError, "Task envelope must be a Hash" unless envelope.is_a?(Hash)
+
+      item = @config.default_task_options.merge(Utilities.stringify_hash_keys(envelope))
+      task = item["task"]
+      arguments = item["arguments"]
+      unless task.is_a?(Class) || (task.is_a?(String) && !task.empty?)
+        raise InvalidJobError, "Task envelope requires a non-empty task"
+      end
+      raise InvalidJobError, "Task arguments must be an Array" unless arguments.is_a?(Array)
+
+      item["task"] = task.name if task.is_a?(Class)
+      item["channel"] = String(item.fetch("channel"))
+      item["id"] ||= SecureRandom.uuid
+      item["created_ms"] ||= Utilities.realtime_milliseconds
+      item
+    end
+
+    def intercept(envelope)
+      context = Publication.new(envelope.fetch("task"), envelope, @config.redis_pool)
+      @config.publish_interceptors.call(context) { envelope }
+    end
+
+    def persist_ready(pipeline, envelopes)
+      envelopes.group_by { |envelope| envelope.fetch("channel") }.each do |channel, group|
+        pipeline.call("SADD", Keyspace::CHANNELS, channel)
+        queued_ms = Utilities.realtime_milliseconds
+        encoded = group.map do |envelope|
+          envelope["queued_ms"] = queued_ms
+          JSON.generate(envelope)
+        end
+        pipeline.call("LPUSH", Keyspace.channel(channel), *encoded)
+      end
+    end
+
+    def validate_arguments!(arguments)
+      invalid = invalid_json_value(arguments)
+      return unless invalid
+
+      message = "Task arguments must contain JSON-native values with string hash keys; found #{invalid.inspect}"
+      case @config.on_complex_arguments
+      when :raise then raise InvalidArgumentError, message
+      when :warn then @config.logger.warn(message)
+      end
+    end
+
+    def invalid_json_value(value)
+      case value
+      when nil, true, false, String, Integer, Float
+        nil
+      when Array
+        value.each do |nested|
+          invalid = invalid_json_value(nested)
+          return invalid if invalid
+        end
+        nil
+      when Hash
+        value.each do |key, nested|
+          return key unless key.is_a?(String)
+
+          invalid = invalid_json_value(nested)
+          return invalid if invalid
+        end
+        nil
+      else
+        value
+      end
+    end
+  end
+end

data/lib/solid_jobs/rails_adapter.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  module RailsAdapter
+    class AdapterTask
+      include SolidJobs::Task
+
+      def execute_task(job_data)
+        ::ActiveJob::Base.execute(job_data.merge("provider_job_id" => task_id))
+      end
+    end
+  end
+end