solid-jobs 0.4.2 → 0.5.0

data/lib/solid_jobs/publication_barrier.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module SolidJobs
+  module PublicationBarrier
+    TIMEOUT_MS = 30_000
+    FIELD = "_solid_jobs_publication"
+    Entry = Data.define(:key, :publication, :owner)
+    RELEASE = <<~LUA.freeze
+      local released = 0
+      for index, key in ipairs(KEYS) do
+        if redis.call("get", key) == ARGV[index] then
+          released = released + redis.call("del", key)
+        end
+      end
+      return released
+    LUA
+
+    module_function
+
+    def prepare(publication: SecureRandom.hex(16), owner: SecureRandom.hex(16))
+      Entry.new(Keyspace.publication(publication), publication, owner)
+    end
+
+    def mark(target, entry)
+      target.call("SET", entry.key, entry.owner, "PX", TIMEOUT_MS)
+    end
+
+    def release(redis_pool, entries)
+      entries = Array(entries)
+      return 0 if entries.empty?
+
+      redis_pool.call(
+        "EVAL", RELEASE, entries.size,
+        *entries.map(&:key),
+        *entries.map(&:owner),
+      )
+    end
+
+    def strip(envelope)
+      envelope.delete(FIELD)
+      envelope
+    end
+  end
+end

data/lib/solid_jobs/publisher.rb

@@ -28,7 +28,8 @@ module SolidJobs
       return unless normalized
 
       validate_arguments!(normalized.fetch("arguments"))
-      persist([normalized])
+      barriers = persist([normalized])
+      emit_persisted([normalized], barriers)
       normalized.fetch("id")
     end
 
@@ -80,13 +81,14 @@ module SolidJobs
           end
           intercepted
         end
-        persist(envelopes)
+        barriers = persist(envelopes)
+        emit_persisted(envelopes, barriers)
       end
       ids
     end
 
     def persist(envelopes)
-      return true if envelopes.empty?
+      return [] if envelopes.empty?
 
       if defined?(Lab)
         case Lab.mode
@@ -94,26 +96,35 @@ module SolidJobs
           envelopes.each do |envelope|
             Lab.captured_for(Utilities.constantize(envelope.fetch("task"))) << envelope
           end
-          return true
+          return []
         when :execute
           envelopes.each { |envelope| Lab.execute(envelope) }
-          return true
+          return []
         end
       end
 
+      planned, ready = envelopes.partition { |envelope| envelope["run_at"] }
+      barriers = []
       @config.redis_pool.pipelined do |pipeline|
-        planned, ready = envelopes.partition { |envelope| envelope["run_at"] }
         planned.each do |envelope|
           score = envelope.delete("run_at")
           pipeline.call("ZADD", Keyspace::PLANNED, score, JSON.generate(envelope))
         end
-        persist_ready(pipeline, ready)
+        barriers = persist_ready(pipeline, ready)
       end
-      true
+      barriers
     end
 
     private
 
+    def emit_persisted(envelopes, barriers)
+      envelopes.each do |envelope|
+        Instrumentation.emit(@config, :enqueued, envelope)
+        Instrumentation.emit(@config, :journaled, envelope)
+      end
+      release_publications(barriers)
+    end
+
     def normalize(envelope)
       raise InvalidJobError, "Task envelope must be a Hash" unless envelope.is_a?(Hash)
 
@@ -138,15 +149,33 @@ module SolidJobs
     end
 
     def persist_ready(pipeline, envelopes)
+      barriers = []
       envelopes.group_by { |envelope| envelope.fetch("channel") }.each do |channel, group|
         pipeline.call("SADD", Keyspace::CHANNELS, channel)
         queued_ms = Utilities.realtime_milliseconds
         encoded = group.map do |envelope|
           envelope["queued_ms"] = queued_ms
+          envelope[PublicationBarrier::FIELD] = SecureRandom.hex(16)
           JSON.generate(envelope)
         end
+        group.zip(encoded).each do |envelope, _payload|
+          barrier = PublicationBarrier.prepare(
+            publication: envelope.fetch(PublicationBarrier::FIELD),
+          )
+          PublicationBarrier.mark(pipeline, barrier)
+          barriers << barrier
+        end
         pipeline.call("LPUSH", Keyspace.channel(channel), *encoded)
       end
+      barriers
+    end
+
+    def release_publications(barriers)
+      PublicationBarrier.release(@config.redis_pool, barriers)
+    rescue SolidRedis::ConnectionError, SolidRedis::CommandError => error
+      @config.logger.warn(
+        "SolidJobs publication barrier release failed; waiting for expiry: #{error.class}: #{error.message}",
+      )
     end
 
     def validate_arguments!(arguments)

data/lib/solid_jobs/recovery.rb

@@ -10,8 +10,21 @@ module SolidJobs
       local payload = redis.call("rpop", KEYS[1])
       if payload then
         local envelope = cjson.decode(payload)
+        local publication = envelope["#{PublicationBarrier::FIELD}"]
+        if not publication then
+          publication = ARGV[3]
+          envelope["#{PublicationBarrier::FIELD}"] = publication
+          payload = cjson.encode(envelope)
+        end
+        redis.call(
+          "set",
+          ARGV[1] .. publication,
+          ARGV[4],
+          "PX",
+          ARGV[2]
+        )
         redis.call("rpush", "#{Keyspace::PREFIX}:channel:" .. envelope["channel"], payload)
-        return payload
+        return {payload, publication, ARGV[4]}
       end
     LUA
 
@@ -22,8 +35,15 @@ module SolidJobs
         identity = node_key.delete_prefix("#{Keyspace::PREFIX}:node:")
         next if alive?(identity)
 
-        while @config.redis_pool.call("EVAL", RESTORE_ONE, 1, key)
+        while (restored = @config.redis_pool.call(
+          "EVAL", RESTORE_ONE, 1, key,
+          Keyspace::PUBLICATION_PREFIX, PublicationBarrier::TIMEOUT_MS,
+          SecureRandom.hex(16), SecureRandom.hex(16),
+        ))
+          payload, publication, owner = restored
           recovered += 1
+          Instrumentation.emit(@config, :recovered, payload)
+          release_publication(publication, owner)
         end
         @config.redis_pool.call("DEL", key)
         @config.redis_pool.call("DEL", Keyspace.claims(identity))
@@ -33,6 +53,15 @@ module SolidJobs
 
     private
 
+    def release_publication(publication, owner)
+      entry = PublicationBarrier.prepare(publication: publication, owner: owner)
+      PublicationBarrier.release(@config.redis_pool, [entry])
+    rescue SolidRedis::ConnectionError, SolidRedis::CommandError => error
+      @config.logger.warn(
+        "SolidJobs recovery barrier release failed; waiting for expiry: #{error.class}: #{error.message}",
+      )
+    end
+
     def alive?(identity)
       hostname, pid = identity.split(":", 3)
       return false if hostname == Socket.gethostname && !process_alive?(Integer(pid))

data/lib/solid_jobs/task.rb

@@ -1,9 +1,44 @@
 # frozen_string_literal: true
 
+require "json"
+
 module SolidJobs
   module Task
+    # Intercepts `enqueue` and reroutes non Ractor-safe tasks to the warm pool.
+    module WarmPoolRouting
+      NON_SHAREABLE_TYPES = [Mutex, Thread, Fiber].freeze
+
+      def enqueue(*arguments)
+        return super unless requires_warm_pool?(arguments)
+
+        require_relative "../../app/jobs/solid_jobs_generic_executor"
+        SolidJobsGenericExecutor.enqueue(
+          SolidJobsGenericExecutor.worker_id_for(name),
+          name,
+          Ractor.make_shareable(JSON.parse(JSON.generate(arguments))),
+        )
+      end
+
+      def requires_warm_pool?(args)
+        return false if equal?(defined?(::SolidJobsGenericExecutor) && ::SolidJobsGenericExecutor)
+        return false unless defined?(::Rails)
+
+        !args.all? { |argument| Ractor.shareable?(argument) } && holds_blocking_state?
+      end
+
+      private
+
+      def holds_blocking_state?
+        instance_variables.any? do |ivar|
+          value = instance_variable_get(ivar)
+          NON_SHAREABLE_TYPES.any? { |type| value.is_a?(type) } || value.class.name.to_s.include?("Pool")
+        end
+      end
+    end
+
     def self.included(base)
       base.extend(TaskMethods)
+      base.singleton_class.prepend(WarmPoolRouting)
       base.include(InstanceMethods)
       base.task_options
     end

data/lib/solid_jobs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SolidJobs
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

data/lib/solid_jobs.rb

@@ -11,6 +11,8 @@ require_relative "solid_jobs/keyspace"
 require_relative "solid_jobs/ractor_support"
 require_relative "solid_jobs/startup_barrier"
 require_relative "solid_jobs/interceptor_registry"
+require_relative "solid_jobs/instrumentation"
+require_relative "solid_jobs/publication_barrier"
 require_relative "solid_jobs/blueprint"
 require_relative "solid_jobs/publisher"
 require_relative "solid_jobs/lab"
@@ -20,6 +22,7 @@ require_relative "solid_jobs/claim"
 require_relative "solid_jobs/failure_policy"
 require_relative "solid_jobs/timer"
 require_relative "solid_jobs/engine"
+require_relative "solid_jobs/observations"
 require_relative "solid_jobs/heartbeat"
 require_relative "solid_jobs/recovery"
 require_relative "solid_jobs/integrity_check"
@@ -44,6 +47,14 @@ module SolidJobs
     config
   end
 
+  def instrumenter
+    config.instrumenter
+  end
+
+  def instrumenter=(value)
+    config.instrumenter = value
+  end
+
   def use_config(configuration)
     Ractor.current[CONFIG_KEY]&.close
     Ractor.current[CONFIG_KEY] = configuration

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solid-jobs
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Nicolas Vandenbogaerde
@@ -129,10 +129,13 @@ files:
 - lib/solid_jobs/executor.rb
 - lib/solid_jobs/failure_policy.rb
 - lib/solid_jobs/heartbeat.rb
+- lib/solid_jobs/instrumentation.rb
 - lib/solid_jobs/integrity_check.rb
 - lib/solid_jobs/interceptor_registry.rb
 - lib/solid_jobs/keyspace.rb
 - lib/solid_jobs/lab.rb
+- lib/solid_jobs/observations.rb
+- lib/solid_jobs/publication_barrier.rb
 - lib/solid_jobs/publisher.rb
 - lib/solid_jobs/ractor_support.rb
 - lib/solid_jobs/rails_adapter.rb