solid-jobs 0.1.3 → 0.2.0

data/lib/solid_jobs/server.rb

@@ -163,7 +163,7 @@ module SolidJobs
           started = SolidJobs::StartupBarrier.boot(ready) do
             local_config = SolidJobs::Config.from_ractor_snapshot(settings)
             SolidJobs.use_config(local_config)
-            scheduler = SolidJobs::Scheduler.new(local_config)
+            scheduler = SolidJobs::Timer.new(local_config)
             local_config.redis_pool.call("PING")
           end
           next :aborted unless started
@@ -265,7 +265,7 @@ module SolidJobs
     end
 
     def remote_signal
-      config.redis_pool.call("RPOP", "#{identity}-signals")
+      config.redis_pool.call("RPOP", Keyspace.node_signals(identity))
     end
 
   end

data/lib/solid_jobs/task.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module SolidJobs
+  module Task
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.include(InstanceMethods)
+      base.task_options
+    end
+
+    module InstanceMethods
+      attr_accessor :task_id
+      attr_writer :_context
+
+      def _context
+        @_context ||= {}
+      end
+
+      def logger
+        SolidJobs.config.logger
+      end
+
+      def interrupted?
+        _context[:interrupted]&.call || false
+      end
+    end
+
+    module ClassMethods
+      def task_options(options = nil)
+        if options
+          merged = current_task_options.merge(Utilities.stringify_keys(options))
+          @task_options = Utilities.shareable_copy(merged)
+        else
+          @task_options ||= Utilities.shareable_copy(SolidJobs.config.default_task_options)
+        end
+      end
+
+      def current_task_options
+        task_options
+      end
+
+      def retry_delay(&block)
+        define_singleton_method(:retry_delay_callback, &block) if block
+        method(:retry_delay_callback) if respond_to?(:retry_delay_callback)
+      end
+
+      def after_final_failure(&block)
+        define_singleton_method(:final_failure_callback, &block) if block
+        method(:final_failure_callback) if respond_to?(:final_failure_callback)
+      end
+
+      def enqueue(*arguments)
+        publish("task" => self, "arguments" => arguments)
+      end
+
+      def enqueue_many(argument_sets, chunk_size: Publisher::DEFAULT_CHUNK_SIZE)
+        Publisher.new.publish_many(
+          current_task_options.merge(
+            "task" => self,
+            "arguments" => argument_sets,
+            "chunk_size" => chunk_size,
+          ),
+        )
+      end
+
+      def enqueue_after(delay, *arguments)
+        enqueue_at(Time.now.to_f + Float(delay), *arguments)
+      end
+
+      def enqueue_at(time, *arguments)
+        timestamp = time.is_a?(Time) ? time.to_f : Float(time)
+        publish("task" => self, "arguments" => arguments, "run_at" => timestamp)
+      end
+
+      def execute(*arguments)
+        Testing.execute(current_task_options.merge("task" => self, "arguments" => arguments))
+      end
+
+      def with_options(options)
+        Submission.new(self, options)
+      end
+
+      def captured
+        Testing.captured_for(self)
+      end
+
+      def clear_captured
+        captured.clear
+      end
+
+      def execute_next
+        envelope = captured.shift
+        raise EmptyQueueError, "No captured task for #{name}" unless envelope
+
+        Testing.execute(envelope)
+      end
+
+      private
+
+      def publish(envelope)
+        Publisher.new.publish(current_task_options.merge(envelope))
+      end
+    end
+
+    class Submission
+      def initialize(task, options)
+        @task = task
+        @options = Utilities.stringify_keys(options)
+      end
+
+      def enqueue(*arguments)
+        return @task.execute(*arguments) if @options.delete("execute")
+
+        if (delay = @options.delete("delay"))
+          enqueue_after(delay, *arguments)
+        elsif (time = @options.delete("run_at"))
+          enqueue_at(time, *arguments)
+        else
+          publish(arguments)
+        end
+      end
+
+      def enqueue_after(delay, *arguments)
+        enqueue_at(Time.now.to_f + Float(delay), *arguments)
+      end
+
+      def enqueue_at(time, *arguments)
+        timestamp = time.is_a?(Time) ? time.to_f : Float(time)
+        publish(arguments, "run_at" => timestamp)
+      end
+
+      private
+
+      def publish(arguments, extra = {})
+        Publisher.new.publish(
+          @task.current_task_options.merge(@options).merge(
+            extra,
+            "task" => @task,
+            "arguments" => arguments,
+          ),
+        )
+      end
+    end
+  end
+
+  class EmptyQueueError < Error; end
+end

data/lib/solid_jobs/testing.rb

@@ -1,9 +1,11 @@
 # frozen_string_literal: true
 
+require "securerandom"
+
 module SolidJobs
   module Testing
-    MODES = %i[disable fake inline].freeze
-    STORAGE_KEY = :solid_jobs_testing_jobs
+    MODES = %i[disable capture execute].freeze
+    STORAGE_KEY = :solid_jobs_testing_captured
     MODE_KEY = :solid_jobs_testing_mode
 
     module_function
@@ -29,14 +31,14 @@ module SolidJobs
       end
     end
 
-    def fake!
-      testing!(:fake) { yield } if block_given?
-      testing!(:fake) unless block_given?
+    def capture!
+      testing!(:capture) { yield } if block_given?
+      testing!(:capture) unless block_given?
     end
 
-    def inline!
-      testing!(:inline) { yield } if block_given?
-      testing!(:inline) unless block_given?
+    def execute!
+      testing!(:execute) { yield } if block_given?
+      testing!(:execute) unless block_given?
     end
 
     def disable!
@@ -44,22 +46,20 @@ module SolidJobs
       testing!(:disable) unless block_given?
     end
 
-    def jobs_for(job_class)
-      storage[job_class.name] ||= []
+    def captured_for(task)
+      storage[task.name] ||= []
     end
 
     def clear_all
       storage.clear
     end
 
-    def execute_inline(payload)
-      job_class = payload["class"]
-      job_class = Utilities.constantize(job_class) if job_class.is_a?(String)
-      instance = job_class.new
-      instance.jid = payload["jid"] if instance.respond_to?(:jid=)
-      SolidJobs.config.server_middleware.invoke(instance, payload, payload.fetch("queue", "default")) do
-        instance.perform(*payload.fetch("args"))
-      end
+    def execute(envelope)
+      normalized = Utilities.stringify_hash_keys(envelope)
+      normalized["task"] = normalized.fetch("task").name if normalized["task"].is_a?(Class)
+      normalized["channel"] ||= "default"
+      normalized["id"] ||= SecureRandom.uuid
+      Executor.new(redis_config: SolidJobs.config.redis_config).execute(normalized)
     end
 
     def storage

data/lib/solid_jobs/{scheduler.rb → timer.rb}

@@ -3,8 +3,8 @@
 require "json"
 
 module SolidJobs
-  class Scheduler
-    SETS = %w[retry schedule].freeze
+  class Timer
+    COLLECTIONS = [Keyspace::RETRIES, Keyspace::PLANNED].freeze
     POP_DUE = <<~LUA.freeze
       local jobs = redis.call("zrange", KEYS[1], "-inf", ARGV[1], "byscore", "limit", 0, 1)
       if jobs[1] then
@@ -15,14 +15,14 @@ module SolidJobs
 
     def initialize(config)
       @config = config
-      @client = Client.new(config: config)
+      @publisher = Publisher.new(config: config)
     end
 
     def enqueue_due(now = Time.now.to_f)
       count = 0
-      SETS.each do |set|
-        while (raw = pop_due(set, now))
-          @client.push(JSON.parse(raw))
+      COLLECTIONS.each do |collection|
+        while (raw = pop_due(collection, now))
+          @publisher.publish(JSON.parse(raw))
           count += 1
         end
       end
@@ -45,8 +45,8 @@ module SolidJobs
 
     private
 
-    def pop_due(set, now)
-      @config.redis_pool.call("EVAL", POP_DUE, 1, set, now)
+    def pop_due(collection, now)
+      @config.redis_pool.call("EVAL", POP_DUE, 1, collection, now)
     end
 
     def randomized_interval

data/lib/solid_jobs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SolidJobs
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end

data/lib/solid_jobs.rb

@@ -7,17 +7,18 @@ require "solid_redis"
 require_relative "solid_jobs/version"
 require_relative "solid_jobs/errors"
 require_relative "solid_jobs/utilities"
+require_relative "solid_jobs/keyspace"
 require_relative "solid_jobs/ractor_support"
 require_relative "solid_jobs/startup_barrier"
-require_relative "solid_jobs/middleware/chain"
+require_relative "solid_jobs/interceptor_registry"
 require_relative "solid_jobs/config"
-require_relative "solid_jobs/client"
+require_relative "solid_jobs/publisher"
 require_relative "solid_jobs/testing"
-require_relative "solid_jobs/job"
-require_relative "solid_jobs/worker"
-require_relative "solid_jobs/fetch"
-require_relative "solid_jobs/retry_service"
-require_relative "solid_jobs/scheduler"
+require_relative "solid_jobs/task"
+require_relative "solid_jobs/executor"
+require_relative "solid_jobs/claim"
+require_relative "solid_jobs/failure_policy"
+require_relative "solid_jobs/timer"
 require_relative "solid_jobs/processor"
 require_relative "solid_jobs/heartbeat"
 require_relative "solid_jobs/recovery"
@@ -57,22 +58,22 @@ module SolidJobs
     Testing.testing!(mode, &block)
   end
 
-  def enqueue(job_class, args, queue: "default", **options)
-    Client.push(
+  def enqueue(task, arguments, channel: "default", **options)
+    Publisher.publish(
       options.merge(
-        "class" => job_class,
-        "args" => args,
-        "queue" => queue,
+        "task" => task,
+        "arguments" => arguments,
+        "channel" => channel,
       ),
     )
   end
 
-  def enqueue_bulk(job_class, arguments, queue: "default", **options)
-    Client.push_bulk(
+  def enqueue_many(task, argument_sets, channel: "default", **options)
+    Publisher.publish_many(
       options.merge(
-        "class" => job_class,
-        "args" => arguments,
-        "queue" => queue,
+        "task" => task,
+        "arguments" => argument_sets,
+        "channel" => channel,
       ),
     )
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solid-jobs
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Nicolas Vandenbogaerde
@@ -105,8 +105,8 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '13.0'
-description: Ractor-local job clients and workers with Sidekiq-compatible Redis payloads,
-  queues, scheduling, and retries.
+description: A Ractor-local Redis task runtime with its own envelope, channels, claims,
+  interceptors, scheduling, and failure handling.
 executables:
 - solid-jobs
 extensions: []
@@ -121,27 +121,28 @@ files:
 - lib/solid_jobs.rb
 - lib/solid_jobs/active_job.rb
 - lib/solid_jobs/api.rb
+- lib/solid_jobs/claim.rb
 - lib/solid_jobs/cli.rb
-- lib/solid_jobs/client.rb
 - lib/solid_jobs/config.rb
 - lib/solid_jobs/errors.rb
-- lib/solid_jobs/fetch.rb
+- lib/solid_jobs/executor.rb
+- lib/solid_jobs/failure_policy.rb
 - lib/solid_jobs/heartbeat.rb
 - lib/solid_jobs/integrity_check.rb
-- lib/solid_jobs/job.rb
-- lib/solid_jobs/middleware/chain.rb
+- lib/solid_jobs/interceptor_registry.rb
+- lib/solid_jobs/keyspace.rb
 - lib/solid_jobs/processor.rb
+- lib/solid_jobs/publisher.rb
 - lib/solid_jobs/ractor_support.rb
 - lib/solid_jobs/rails.rb
 - lib/solid_jobs/recovery.rb
-- lib/solid_jobs/retry_service.rb
-- lib/solid_jobs/scheduler.rb
 - lib/solid_jobs/server.rb
 - lib/solid_jobs/startup_barrier.rb
+- lib/solid_jobs/task.rb
 - lib/solid_jobs/testing.rb
+- lib/solid_jobs/timer.rb
 - lib/solid_jobs/utilities.rb
 - lib/solid_jobs/version.rb
-- lib/solid_jobs/worker.rb
 homepage: https://github.com/nicolasva/solid-jobs
 licenses:
 - MIT

data/lib/solid_jobs/client.rb

@@ -1,202 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require "securerandom"
-
-module SolidJobs
-  class Client
-    DEFAULT_BATCH_SIZE = 1_000
-    SCHEDULED_BATCH_SIZE = 100
-
-    def self.push(item)
-      new.push(item)
-    end
-
-    def self.push_bulk(item)
-      new.push_bulk(item)
-    end
-
-    def initialize(config: nil, configuration: nil)
-      selected = config || configuration || SolidJobs.config
-      @config = selected.is_a?(SolidRedis::Config) ? Config.new(redis: selected) : selected
-    end
-
-    def push(item)
-      payload = normalize_item(item)
-      payload = invoke_middleware(payload)
-      return unless payload
-
-      verify_json!(payload)
-      raw_push([payload])
-      payload["jid"]
-    end
-
-    def push_bulk(item)
-      raise InvalidJobError, "Job payload must be a Hash" unless item.is_a?(Hash)
-
-      source = Utilities.stringify_hash_keys(item)
-      args = source.delete("args")
-      raise InvalidJobError, "Job 'args' must be an Array or Enumerable" unless args.respond_to?(:each)
-
-      ats = source.delete("at")
-      if ats && !ats.is_a?(Numeric) && !ats.is_a?(Array)
-        raise InvalidJobError, "Job 'at' must be a Numeric or an Array"
-      end
-      if ats.is_a?(Array) && args.respond_to?(:size) && ats.size != args.size
-        raise InvalidJobError, "Job 'at' Array must have the same size as 'args'"
-      end
-      batch_size = Integer(source.delete("batch_size") || (ats ? SCHEDULED_BATCH_SIZE : DEFAULT_BATCH_SIZE))
-      raise ArgumentError, "batch_size must be positive" unless batch_size.positive?
-
-      explicit_jid = source.delete("jid")
-      if explicit_jid && args.respond_to?(:size) && args.size > 1
-        raise InvalidJobError, "Explicit jid is only supported for one bulk job"
-      end
-      base = normalize_item(source.merge("args" => []))
-      base.delete("jid")
-      result = []
-      args.each_slice(batch_size).with_index do |batch, batch_index|
-        payloads = []
-        batch.each_with_index do |job_args, index|
-          raise InvalidJobError, "Bulk job arguments must be Arrays" unless job_args.is_a?(Array)
-
-          payload = base.merge(
-            "args" => job_args,
-            "jid" => explicit_jid || SecureRandom.hex(12),
-          )
-          payload["at"] = ats.is_a?(Array) ? ats[batch_index * batch_size + index] : ats if ats
-          normalized = invoke_middleware(payload)
-          if normalized
-            verify_json!(normalized)
-            payloads << normalized
-            result << normalized["jid"]
-          else
-            result << nil
-          end
-        end
-        raw_push(payloads)
-      end
-      result
-    end
-
-    def raw_push(payloads)
-      return true if payloads.empty?
-
-      if defined?(Testing)
-        case Testing.mode
-        when :fake
-          payloads.each do |payload|
-            Testing.jobs_for(Utilities.constantize(payload["class"])) << payload
-          end
-          return true
-        when :inline
-          payloads.each { |payload| Testing.execute_inline(payload) }
-          return true
-        end
-      end
-
-      @config.redis_pool.pipelined do |pipeline|
-        if payloads.all? { |payload| payload["at"] }
-          payloads.each do |payload|
-            score = payload.delete("at")
-            pipeline.call("ZADD", "schedule", score, JSON.generate(payload))
-          end
-        elsif payloads.none? { |payload| payload["at"] }
-          push_immediate(pipeline, payloads)
-        else
-          scheduled, immediate = payloads.partition { |payload| payload["at"] }
-          scheduled.each do |payload|
-            score = payload.delete("at")
-            pipeline.call("ZADD", "schedule", score, JSON.generate(payload))
-          end
-          push_immediate(pipeline, immediate)
-        end
-      end
-      true
-    end
-
-    private
-
-    def normalize_item(item)
-      raise InvalidJobError, "Job payload must be a Hash" unless item.is_a?(Hash)
-
-      payload = @config.default_job_options.merge(Utilities.stringify_hash_keys(item))
-      payload["class"] ||= payload.delete("job_class")
-      payload["jid"] ||= payload.delete("job_id")
-      job_class = payload["class"]
-      args = payload["args"]
-      unless job_class.is_a?(Class) || (job_class.is_a?(String) && !job_class.empty?)
-        raise InvalidJobError, "Job must include a non-empty class"
-      end
-      raise InvalidJobError, "Job args must be an Array" unless args.is_a?(Array)
-
-      payload["class"] = job_class.name if job_class.is_a?(Class)
-      payload["queue"] = String(payload.fetch("queue"))
-      payload["jid"] ||= SecureRandom.hex(12)
-      payload["created_at"] ||= Utilities.realtime_milliseconds
-      payload
-    end
-
-    def invoke_middleware(payload)
-      job_class = payload["class"]
-      @config.client_middleware.invoke(job_class, payload, payload["queue"], @config.redis_pool) do
-        payload
-      end
-    end
-
-    def push_immediate(pipeline, payloads)
-      return if payloads.empty?
-
-      groups = if payloads.all? { |payload| payload["queue"] == payloads.first["queue"] }
-        [[payloads.first["queue"], payloads]]
-      else
-        payloads.group_by { |payload| payload["queue"] }
-      end
-      groups.each do |queue, jobs|
-        pipeline.call("SADD", "queues", queue)
-        now = Utilities.realtime_milliseconds
-        encoded = jobs.map do |payload|
-          payload["enqueued_at"] = now
-          JSON.generate(payload)
-        end
-        pipeline.call("LPUSH", "queue:#{queue}", *encoded)
-      end
-    end
-
-    def verify_json!(payload)
-      invalid = find_invalid_json_value(payload["args"])
-      return unless invalid
-
-      message = "Job arguments must contain only JSON-native values with string hash keys; found #{invalid.inspect}"
-      case @config.on_complex_arguments
-      when :raise
-        raise InvalidArgumentError, message
-      when :warn
-        @config.logger.warn(message)
-      end
-    end
-
-    def find_invalid_json_value(value)
-      case value
-      when nil, true, false, String, Integer, Float
-        nil
-      when Array
-        value.each do |nested|
-          invalid = find_invalid_json_value(nested)
-          return invalid if invalid
-        end
-        nil
-      when Hash
-        value.each do |key, nested|
-          return key unless key.is_a?(String)
-
-          invalid = find_invalid_json_value(nested)
-          return invalid if invalid
-        end
-        nil
-      else
-        value
-      end
-    end
-  end
-end