solid-jobs 0.4.2 → 0.5.0

data/lib/solid_jobs/conductor.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "securerandom"
-require "socket"
 require "timeout"
 
 module SolidJobs
@@ -20,7 +18,7 @@ module SolidJobs
       @scheduler = nil
       @heartbeat = nil
       @started = false
-      @identity = "#{Socket.gethostname}:#{::Process.pid}:#{SecureRandom.hex(6)}".freeze
+      @identity = config.identity
       @started_at = Time.now.to_f
     end
 
@@ -29,7 +27,7 @@ module SolidJobs
 
       warn_ruby34_multi_ractor
       snapshot = Utilities.shareable_copy(
-        config.ractor_snapshot.merge(identity: identity, started_at: @started_at),
+        config.ractor_snapshot.merge(started_at: @started_at),
       )
       Recovery.call(config: config)
       barrier = StartupBarrier.new(config.concurrency + 2)
@@ -60,6 +58,7 @@ module SolidJobs
           quiet = false
           stopping = false
           work = {}
+          redis = {}
           processed = failed = 0
           next_beat = 0.0
           until stopping
@@ -68,6 +67,7 @@ module SolidJobs
                 case (message = messages.pop(true))
                 when :quiet
                   quiet = true
+                  heartbeat.observe(quiet: true, work: work, redis: redis)
                 when :stop
                   stopping = true
                 else
@@ -75,11 +75,21 @@ module SolidJobs
                   case event
                   when :work
                     work[arguments[0].to_s] = arguments[1]
+                    heartbeat.observe_ractor(
+                      arguments[0],
+                      arguments[1],
+                      job_id: arguments[2],
+                      quiet: quiet,
+                    )
                   when :done
                     work.delete(arguments[0].to_s)
+                    heartbeat.observe_ractor(arguments[0], nil, quiet: quiet)
                   when :stats
                     processed += arguments[0]
                     failed += arguments[1]
+                  when :redis
+                    redis[arguments[0].to_s] = arguments[1]
+                    heartbeat.observe_redis(arguments[0], arguments[1])
                   end
                 end
               end
@@ -89,7 +99,13 @@ module SolidJobs
             now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
             if now >= next_beat
               begin
-                heartbeat.beat(quiet: quiet, work: work, processed: processed, failed: failed)
+                heartbeat.beat(
+                  quiet: quiet,
+                  work: work,
+                  processed: processed,
+                  failed: failed,
+                  redis: redis,
+                )
                 processed = failed = 0
               rescue SolidRedis::ConnectionError => error
                 local_config.logger.warn("Redis heartbeat failed, retrying: #{error.class}: #{error.message}")
@@ -99,7 +115,15 @@ module SolidJobs
             sleep 0.05 unless stopping
           end
           begin
-            heartbeat.beat(quiet: quiet, work: work, processed: processed, failed: failed)
+            heartbeat.observe(quiet: quiet, work: work, redis: redis, phase: :stopping)
+            heartbeat.beat(
+              quiet: quiet,
+              work: work,
+              processed: processed,
+              failed: failed,
+              redis: redis,
+              phase: :stopped,
+            )
             heartbeat.cleanup
           rescue SolidRedis::ConnectionError => error
             local_config.logger.warn("Redis heartbeat cleanup failed: #{error.class}: #{error.message}")

data/lib/solid_jobs/engine.rb

@@ -13,6 +13,7 @@ module SolidJobs
       @heartbeat = heartbeat
       @state_mutex = Mutex.new
       @busy = false
+      @redis_status = nil
     end
 
     def run(control:)
@@ -44,18 +45,40 @@ module SolidJobs
           @state_mutex.synchronize { @busy = true }
           envelope = claim.envelope
           register_work(claim, envelope)
+          attributes = telemetry_attributes(claim)
+          Instrumentation.emit(@config, :started, envelope, **attributes)
+          started_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
           begin
             @executor.execute(envelope)
           rescue ExecutionHalt
-            requeue(claim)
-            break
+            raise
           rescue StandardError => error
-            FailurePolicy.call(config: @config, payload: envelope, error: error)
+            duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started_at
+            Instrumentation.emit(
+              @config,
+              :failed,
+              envelope,
+              **attributes,
+              duration: duration,
+              error_class: error.class.name,
+              error_message: error.message.to_s,
+            )
+            FailurePolicy.call(
+              config: @config,
+              payload: envelope,
+              error: error,
+              telemetry_attributes: attributes,
+            )
             complete(claim)
             failed += 1
           else
+            duration = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - started_at
+            Instrumentation.emit(@config, :completed, envelope, **attributes, duration: duration)
             processed += 1 if complete(claim)
           end
+        rescue ExecutionHalt
+          requeue(claim)
+          break
         ensure
           @state_mutex.synchronize { @busy = false }
           clear_work
@@ -80,14 +103,24 @@ module SolidJobs
 
     def complete(claim)
       completed = claim.complete
+      report_redis("connected")
       unless completed
         @config.logger.warn(
           "Claim fencing rejected stale completion: #{claim.claim_token}",
         )
       end
+      if completed
+        Instrumentation.emit(
+          @config,
+          :acknowledged,
+          claim.envelope,
+          **telemetry_attributes(claim),
+        )
+      end
       completed
     rescue SolidRedis::ConnectionError => error
-      @fetcher.connection_failed!
+      report_redis("disconnected")
+      @claims.connection_failed!
       @config.logger.warn(
         "Redis completion failed; claimed task will be replayed: #{error.class}: #{error.message}",
       )
@@ -96,17 +129,24 @@ module SolidJobs
 
     def requeue(claim)
       requeued = claim.requeue
+      report_redis("connected")
       unless requeued
         @config.logger.warn(
           "Claim fencing rejected stale requeue: #{claim.claim_token}",
         )
       end
       requeued
+    rescue SolidRedis::ConnectionError
+      report_redis("disconnected")
+      raise
     end
 
     def retrieve
-      @claims.next
+      claim = @claims.next
+      report_redis("connected")
+      claim
     rescue SolidRedis::ConnectionError => error
+      report_redis("disconnected")
       @claims.connection_failed!
       @config.logger.warn("Redis claim failed, retrying: #{error.class}: #{error.message}")
       sleep 0.1
@@ -117,13 +157,14 @@ module SolidJobs
       return unless @identity
 
       work = JSON.generate(
+        "job_id" => envelope["id"],
         "channel" => claim.channel,
         "envelope" => envelope,
         "started_at" => Time.now.to_f,
         "claim_token" => claim.claim_token,
         "attempt" => claim.attempt,
       )
-      send_heartbeat(:work, @processor_id, work)
+      send_heartbeat(:work, @processor_id, work, envelope["id"])
     end
 
     def clear_work
@@ -133,7 +174,14 @@ module SolidJobs
     end
 
     def send_heartbeat(*message)
-      @heartbeat&.send(message, move: true)
+      @heartbeat&.send(Utilities.shareable_copy(message))
+    end
+
+    def report_redis(status)
+      return if @redis_status == status
+
+      @redis_status = status
+      send_heartbeat(:redis, @processor_id, status)
     end
 
     def receive_commands(control)
@@ -148,5 +196,13 @@ module SolidJobs
     rescue ThreadError
       nil
     end
+
+    def telemetry_attributes(claim)
+      {
+        reservation_id: claim.claim_token,
+        attempt: claim.attempt,
+        **Instrumentation.worker_attributes(@processor_id),
+      }
+    end
   end
 end

data/lib/solid_jobs/failure_policy.rb

@@ -19,11 +19,22 @@ module SolidJobs
 
       if retry_allowed?(payload, failure_count, now)
         decision = retry_decision(payload, failure_count)
-        return nil if decision == :drop
+        if decision == :drop
+          emit_dead(payload)
+          return nil
+        end
         return archive(payload, now.to_f) if decision == :archive
 
         next_attempt = now.to_f + (decision || retry_delay(failure_count))
         @config.redis_pool.call("ZADD", Keyspace::RETRIES, next_attempt, JSON.generate(payload))
+        Instrumentation.emit(
+          @config,
+          :retry_scheduled,
+          payload,
+          **telemetry_attributes,
+          error_class: @error.class.name,
+          error_message: @error.message.to_s,
+        )
         append_message("retry scheduled")
         next_attempt
       else
@@ -36,6 +47,10 @@ module SolidJobs
 
     private
 
+    def telemetry_attributes
+      @telemetry_attributes || {}
+    end
+
     def retry_allowed?(payload, failure_count, now)
       if payload["retry_within"]
         first_failure = Float(payload["first_failed_ms"]) / 1_000
@@ -78,7 +93,7 @@ module SolidJobs
     end
 
     def archive(payload, timestamp)
-      @config.redis_pool.pipelined do |pipeline|
+      result = @config.redis_pool.pipelined do |pipeline|
         pipeline.call("ZADD", Keyspace::DISCARDED, timestamp, JSON.generate(payload))
         pipeline.call(
           "ZREMRANGEBYSCORE",
@@ -93,6 +108,19 @@ module SolidJobs
           -(@config.discarded_limit + 1),
         )
       end
+      emit_dead(payload)
+      result
+    end
+
+    def emit_dead(payload)
+      Instrumentation.emit(
+        @config,
+        :dead,
+        payload,
+        **telemetry_attributes,
+        error_class: @error.class.name,
+        error_message: @error.message.to_s,
+      )
     end
   end
 end

data/lib/solid_jobs/heartbeat.rb

@@ -8,14 +8,15 @@ module SolidJobs
     INTERVAL = 5
     TTL = 60
 
-    def initialize(config, identity:, started_at:, concurrency:)
+    def initialize(config, identity:, started_at:, concurrency:, observations: Observations.new)
       @config = config
       @identity = identity
       @started_at = started_at
       @concurrency = concurrency
+      @observations = observations
     end
 
-    def beat(quiet: false, work: {}, processed: 0, failed: 0)
+    def beat(quiet: false, work: {}, processed: 0, failed: 0, redis: {}, phase: nil)
       now = Time.now.to_f
       info = {
         "hostname" => Socket.gethostname,
@@ -27,28 +28,81 @@ module SolidJobs
       }
       node_key = Keyspace.node(@identity)
       work_key = Keyspace.node_work(@identity)
-      busy = @config.redis_pool.call("HLEN", work_key)
-      @config.redis_pool.pipelined do |pipeline|
-        pipeline.call("INCRBY", Keyspace::PROCESSED, processed) if processed.positive?
-        pipeline.call("INCRBY", Keyspace::FAILED, failed) if failed.positive?
-        pipeline.call("SADD", Keyspace::NODES, @identity)
-        pipeline.call(
-          "HSET",
-          node_key,
-          "info", JSON.generate(info),
-          "beat", now,
-          "busy", busy,
-          "quiet", quiet ? "true" : "false",
-          "concurrency", @concurrency,
-        )
-        pipeline.call("EXPIRE", node_key, TTL)
-        pipeline.call("DEL", work_key)
-        pipeline.call("HSET", work_key, *work.flatten(1)) unless work.empty?
-        pipeline.call("EXPIRE", work_key, TTL)
+      begin
+        busy = @config.redis_pool.call("HLEN", work_key)
+        @config.redis_pool.pipelined do |pipeline|
+          pipeline.call("INCRBY", Keyspace::PROCESSED, processed) if processed.positive?
+          pipeline.call("INCRBY", Keyspace::FAILED, failed) if failed.positive?
+          pipeline.call("SADD", Keyspace::NODES, @identity)
+          pipeline.call(
+            "HSET",
+            node_key,
+            "info", JSON.generate(info),
+            "beat", now,
+            "busy", busy,
+            "quiet", quiet ? "true" : "false",
+            "concurrency", @concurrency,
+          )
+          pipeline.call("EXPIRE", node_key, TTL)
+          pipeline.call("DEL", work_key)
+          pipeline.call("HSET", work_key, *work.flatten(1)) unless work.empty?
+          pipeline.call("EXPIRE", work_key, TTL)
+        end
+      ensure
+        observe(quiet: quiet, work: work, redis: redis, phase: phase, observed_at: now)
       end
       now
     end
 
+    def observe(quiet: false, work: {}, redis: {}, phase: nil, observed_at: Time.now.to_f)
+      Instrumentation.observe(
+        @config,
+        "process.observed",
+        @observations.process(node_id: @identity, observed_at: observed_at),
+      )
+      @observations.ractors(
+        node_id: @identity,
+        concurrency: @concurrency,
+        work: work,
+        quiet: quiet,
+        phase: phase,
+        observed_at: observed_at,
+      ).each { |payload| Instrumentation.observe(@config, "ractor.observed", payload) }
+      @observations.redis(
+        node_id: @identity,
+        concurrency: @concurrency,
+        statuses: redis,
+        observed_at: observed_at,
+      ).each { |payload| Instrumentation.observe(@config, "redis.observed", payload) }
+      true
+    end
+
+    def observe_redis(processor_id, status, observed_at: Time.now.to_f)
+      payload = @observations.redis_worker(
+        node_id: @identity,
+        processor_id: processor_id,
+        status: status,
+        observed_at: observed_at,
+      )
+      Instrumentation.observe(@config, "redis.observed", payload)
+    end
+
+    def observe_ractor(
+      processor_id, work, job_id: nil, quiet: false, phase: nil,
+      observed_at: Time.now.to_f
+    )
+      payload = @observations.ractor_worker(
+        node_id: @identity,
+        processor_id: processor_id,
+        work: work,
+        job_id: job_id,
+        quiet: quiet,
+        phase: phase,
+        observed_at: observed_at,
+      )
+      Instrumentation.observe(@config, "ractor.observed", payload)
+    end
+
     def cleanup
       @config.redis_pool.pipelined do |pipeline|
         pipeline.call("SREM", Keyspace::NODES, @identity)

data/lib/solid_jobs/instrumentation.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SolidJobs
+  module NullInstrumenter
+    module_function
+
+    def instrument(_name, _payload = nil)
+      yield if block_given?
+    end
+  end
+
+  module Instrumentation
+    EVENTS = {
+      enqueued: "job.enqueued",
+      reserved: "job.reserved",
+      journaled: "job.journaled",
+      started: "job.started",
+      completed: "job.completed",
+      failed: "job.failed",
+      retry_scheduled: "job.retry_scheduled",
+      acknowledged: "job.acknowledged",
+      recovered: "job.recovered",
+      dead: "job.dead",
+    }.freeze
+
+    module_function
+
+    def emit(config, event, envelope, **attributes)
+      envelope = JSON.parse(envelope) if envelope.is_a?(String)
+      payload = {
+        job_id: envelope["id"],
+        queue: envelope["channel"],
+        job_class: envelope["task"],
+        node_id: config.identity,
+      }.merge(attributes).reject { |_key, value| value.nil? }
+      config.instrumenter.instrument(EVENTS.fetch(event), payload)
+      true
+    rescue ExecutionHalt
+      raise
+    rescue StandardError, ScriptError => error
+      begin
+        config.logger.warn("SolidJobs telemetry failed: #{error.class}: #{error.message}")
+      rescue ExecutionHalt
+        raise
+      rescue StandardError, ScriptError
+        nil
+      end
+      false
+    end
+
+    def worker_attributes(processor_id)
+      return {} if processor_id.nil?
+
+      { ractor_id: processor_id, worker_id: processor_id }
+    end
+
+    def observe(config, name, payload)
+      config.instrumenter.instrument(name, payload)
+      true
+    rescue ExecutionHalt
+      raise
+    rescue StandardError, ScriptError => error
+      begin
+        config.logger.warn("SolidJobs telemetry failed: #{error.class}: #{error.message}")
+      rescue ExecutionHalt
+        raise
+      rescue StandardError, ScriptError
+        nil
+      end
+      false
+    end
+  end
+end

data/lib/solid_jobs/keyspace.rb

@@ -10,6 +10,7 @@ module SolidJobs
     DISCARDED = "#{PREFIX}:discarded".freeze
     NODES = "#{PREFIX}:nodes".freeze
     ATTEMPTS = "#{PREFIX}:attempts".freeze
+    PUBLICATION_PREFIX = "#{PREFIX}:publication:".freeze
     PROCESSED = "#{PREFIX}:metrics:processed".freeze
     FAILED = "#{PREFIX}:metrics:failed".freeze
 
@@ -19,6 +20,10 @@ module SolidJobs
       "#{PREFIX}:channel:#{name}"
     end
 
+    def publication(digest)
+      "#{PUBLICATION_PREFIX}#{digest}"
+    end
+
     def node(identity)
       "#{PREFIX}:node:#{identity}"
     end

data/lib/solid_jobs/observations.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require "json"
+require "open3"
+require "socket"
+
+module SolidJobs
+  class Observations
+    PROCESS_METRICS = Ractor.make_shareable({
+      cpu_time: ["seconds", "ruby-process-clock"],
+      rss: ["bytes", "ps"],
+      gc_count: ["collections", "ruby-gc"],
+      gc_time: ["nanoseconds", "ruby-gc"],
+      allocations: ["objects", "ruby-gc"],
+    })
+    REDIS_METRICS = Ractor.make_shareable({
+      latency: ["seconds", "solid-jobs"],
+      commands: ["commands", "solid-jobs"],
+      memory: ["bytes", "solid-jobs"],
+      connections: ["connections", "solid-jobs"],
+    })
+
+    def initialize(
+      clock: -> { Time.now.to_f },
+      sources: nil
+    )
+      @clock = clock
+      @sources = sources || default_sources
+      @last_success = {}
+    end
+
+    def process(node_id:, observed_at: @clock.call)
+      {
+        node_id: node_id,
+        process_id: node_id,
+        hostname: Socket.gethostname,
+        observed_at: observed_at,
+        metrics: PROCESS_METRICS.to_h do |name, (unit, source)|
+          [name, sample(name, unit, source, observed_at)]
+        end,
+      }
+    end
+
+    def ractors(node_id:, concurrency:, work:, quiet: false, phase: nil, observed_at: @clock.call)
+      Array.new(concurrency) do |processor_id|
+        ractor_worker(
+          node_id: node_id,
+          processor_id: processor_id,
+          work: work[processor_id.to_s],
+          quiet: quiet,
+          phase: phase,
+          observed_at: observed_at,
+        )
+      end
+    end
+
+    def ractor_worker(
+      node_id:, processor_id:, work:, job_id: nil, quiet: false, phase: nil,
+      observed_at: @clock.call
+    )
+      job_id ||= current_job_id(work)
+      state, activity = worker_state(job_id, quiet, phase)
+      {
+        node_id: node_id,
+        observed_at: observed_at,
+        ractor_id: processor_id,
+        worker_id: processor_id,
+        executor_id: processor_id,
+        state: state,
+        activity: activity,
+        current_job_ids: job_id ? [job_id] : [],
+      }
+    end
+
+    def redis(node_id:, concurrency:, statuses:, observed_at: @clock.call)
+      Array.new(concurrency) do |processor_id|
+        redis_worker(
+          node_id: node_id,
+          processor_id: processor_id,
+          status: statuses.fetch(processor_id.to_s, "unavailable"),
+          observed_at: observed_at,
+        )
+      end
+    end
+
+    def redis_worker(node_id:, processor_id:, status:, observed_at: @clock.call)
+      {
+        node_id: node_id,
+        observed_at: observed_at,
+        ractor_id: processor_id,
+        worker_id: processor_id,
+        executor_id: processor_id,
+        connection_status: status,
+        metrics: REDIS_METRICS.to_h do |name, (unit, source)|
+          [name, unavailable(unit, source)]
+        end,
+      }
+    end
+
+    private
+
+    def sample(name, unit, source, observed_at)
+      value = @sources.fetch(name).call
+      raise RangeError, "#{name} must be non-negative" unless value.is_a?(Numeric) && value.finite? && !value.negative?
+
+      @last_success[name] = [value, observed_at]
+      measurement(value, unit, "available", source, observed_at, 0.0)
+    rescue StandardError, NotImplementedError
+      last = @last_success[name]
+      return unavailable(unit, source) unless last
+
+      value, sampled_at = last
+      measurement(value, unit, "stale", source, sampled_at, observed_at - sampled_at)
+    end
+
+    def measurement(value, unit, status, source, observed_at, age_seconds)
+      {
+        value: value,
+        unit: unit,
+        status: status,
+        source: source,
+        observed_at: observed_at,
+        age_seconds: age_seconds,
+      }
+    end
+
+    def unavailable(unit, source)
+      measurement(nil, unit, "unavailable", source, nil, nil)
+    end
+
+    def current_job_id(raw)
+      return unless raw
+
+      record = JSON.parse(raw)
+      direct_job_id = record["job_id"]
+      return direct_job_id if direct_job_id.is_a?(String) && !direct_job_id.empty?
+
+      envelope = record.fetch("envelope")
+      envelope = JSON.parse(envelope) if envelope.is_a?(String)
+      job_id = envelope["id"]
+      job_id if job_id.is_a?(String) && !job_id.empty?
+    rescue JSON::ParserError, KeyError, TypeError
+      nil
+    end
+
+    def worker_state(job_id, quiet, phase)
+      case phase
+      when :stopping then %w[stopping stopping]
+      when :stopped then %w[stopped stopped]
+      else
+        job_id ? %w[busy executing] : ["idle", quiet ? "waiting" : "waiting"]
+      end
+    end
+
+    def default_sources
+      {
+        cpu_time: -> { Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID) },
+        rss: method(:read_rss),
+        gc_count: -> { GC.count },
+        gc_time: lambda {
+          raise NotImplementedError, "GC.total_time unavailable" unless GC.respond_to?(:total_time)
+
+          GC.measure_total_time = true if GC.respond_to?(:measure_total_time=)
+          GC.total_time
+        },
+        allocations: -> { GC.stat.fetch(:total_allocated_objects) },
+      }
+    end
+
+    def read_rss
+      stdout, stderr, status = Open3.capture3("ps", "-o", "rss=", "-p", Process.pid.to_s)
+      raise "ps failed: #{stderr.strip}" unless status.success?
+
+      Integer(stdout.strip) * 1024
+    end
+  end
+end