queue_pulse 0.1.0

data/lib/queue_pulse/monitor.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "timeout"
+
+module QueuePulse
+  # Runs one monitoring pass: evaluate enabled checks, collect alerts, deliver
+  # to every notifier. Heavily isolated — a failing check or notifier is logged
+  # and never propagates to the host app (Requirements US-6.1, US-6.5, US-7.4/5).
+  class Monitor
+    CHECK_CLASSES = {
+      failures: Checks::Failures,
+      queue_latency: Checks::QueueLatency,
+      queue_depth: Checks::QueueDepth,
+      stuck_jobs: Checks::StuckJobs,
+      worker_liveness: Checks::WorkerLiveness
+    }.freeze
+
+    def initialize(config:, source: nil)
+      @config = config
+      @source = source || Source::SolidQueue.new
+      @state = State.new(config.cache)
+    end
+
+    # @return [Array<QueuePulse::Alert>] delivered alerts
+    def run
+      unless @source.available?
+        logger.warn("[QueuePulse] Solid Queue tables not found; skipping checks")
+        return []
+      end
+
+      if @config.notifiers.empty?
+        logger.warn("[QueuePulse] no notifiers configured; skipping checks")
+        return []
+      end
+
+      alerts = Timeout.timeout(@config.check_timeout) { collect_alerts }
+      deliver(alerts)
+      alerts
+    rescue Timeout::Error
+      logger.error("[QueuePulse] check pass exceeded #{@config.check_timeout}s and was aborted")
+      []
+    end
+
+    private
+
+    def collect_alerts
+      Configuration::ALL_CHECKS.each_with_object([]) do |name, alerts|
+        next unless @config.check_enabled?(name)
+
+        klass = CHECK_CLASSES.fetch(name)
+        begin
+          alerts.concat(klass.new(@config, @source, @state).call)
+        rescue StandardError => e
+          logger.error("[QueuePulse] check #{name} failed: #{e.class}: #{e.message}")
+        end
+      end
+    end
+
+    def deliver(alerts)
+      return if alerts.empty?
+
+      @config.notifiers.each do |notifier|
+        notifier.deliver(alerts)
+      rescue StandardError => e
+        logger.error("[QueuePulse] notifier #{notifier.class} failed: #{e.class}: #{e.message}")
+      end
+    end
+
+    def logger
+      @config.logger
+    end
+  end
+end

data/lib/queue_pulse/notifiers/base.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module QueuePulse
+  module Notifiers
+    # Shared behaviour for notifiers. Subclasses implement #deliver(alerts).
+    # HTTP helpers retry up to 2 times on timeout/non-2xx and apply timeouts so
+    # a slow endpoint can't hang a check (Requirements US-6.6).
+    class Base
+      MAX_RETRIES = 2
+      OPEN_TIMEOUT = 5
+      READ_TIMEOUT = 5
+
+      # @param alerts [Array<QueuePulse::Alert>]
+      def deliver(_alerts)
+        raise NotImplementedError
+      end
+
+      protected
+
+      def logger
+        QueuePulse.logger
+      end
+
+      def post_json(url, payload, headers: {})
+        body = JSON.generate(payload)
+        post(url, body: body, headers: { "Content-Type" => "application/json" }.merge(headers))
+      end
+
+      def post(url, body:, headers: {})
+        uri = URI.parse(url)
+        attempt = 0
+        begin
+          attempt += 1
+          response = http_post(uri, body, headers)
+          unless response.is_a?(Net::HTTPSuccess)
+            raise QueuePulse::Error, "HTTP #{response.code} from #{uri.host}"
+          end
+
+          response
+        rescue QueuePulse::Error, Timeout::Error, SystemCallError, SocketError, Net::HTTPBadResponse, OpenSSL::SSL::SSLError => e
+          if attempt <= MAX_RETRIES
+            sleep(retry_backoff(attempt))
+            retry
+          end
+          raise QueuePulse::Error, "delivery to #{uri.host} failed after #{attempt} attempts: #{e.message}"
+        end
+      end
+
+      # Overridable so tests can avoid real sleeping.
+      def retry_backoff(attempt)
+        0.5 * attempt
+      end
+
+      private
+
+      def http_post(uri, body, headers)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = OPEN_TIMEOUT
+        http.read_timeout = READ_TIMEOUT
+
+        request = Net::HTTP::Post.new(uri.request_uri)
+        headers.each { |k, v| request[k] = v }
+        request.body = body
+        http.request(request)
+      end
+    end
+  end
+end

data/lib/queue_pulse/notifiers/email.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module QueuePulse
+  module Notifiers
+    # Sends a plain-text digest email via ActionMailer. Body building is
+    # separated for testing. Requires ActionMailer to be configured in the host
+    # app (Requirements US-6.4).
+    class Email < Base
+      def initialize(to:, from: "queue_pulse@localhost", subject_prefix: "[QueuePulse]")
+        super()
+        @to = to
+        @from = from
+        @subject_prefix = subject_prefix
+      end
+
+      def deliver(alerts)
+        return if alerts.empty?
+
+        unless defined?(ActionMailer::Base)
+          logger.warn("[QueuePulse] Email notifier requires ActionMailer; skipping #{alerts.size} alert(s)")
+          return
+        end
+
+        mail = ActionMailer::Base.mail(
+          to: @to,
+          from: @from,
+          subject: subject(alerts),
+          body: body(alerts)
+        )
+        mail.deliver_now
+      end
+
+      def subject(alerts)
+        if alerts.size == 1
+          "#{@subject_prefix} #{alerts.first.title}"
+        else
+          "#{@subject_prefix} #{alerts.size} alerts"
+        end
+      end
+
+      def body(alerts)
+        alerts.map do |alert|
+          lines = ["#{alert.emoji} #{alert.title}", alert.message]
+          alert.context.each { |k, v| lines << "  #{k}: #{v}" }
+          lines.join("\n")
+        end.join("\n\n")
+      end
+    end
+  end
+end

data/lib/queue_pulse/notifiers/slack.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module QueuePulse
+  module Notifiers
+    # Posts alerts to a Slack Incoming Webhook. Payload building is separated
+    # from delivery so it can be unit-tested without the network.
+    class Slack < Base
+      def initialize(webhook_url:)
+        super()
+        @webhook_url = webhook_url
+      end
+
+      def deliver(alerts)
+        return if alerts.empty?
+
+        post_json(@webhook_url, payload(alerts))
+      end
+
+      # @return [Hash] the Slack message body for these alerts
+      def payload(alerts)
+        { blocks: alerts.flat_map { |alert| blocks_for(alert) } }
+      end
+
+      private
+
+      def blocks_for(alert)
+        detail = alert.context.map { |k, v| "*#{k}:* #{v}" }.join("\n")
+        [
+          {
+            type: "section",
+            text: { type: "mrkdwn", text: "#{alert.emoji} *#{alert.title}*\n#{alert.message}" }
+          },
+          ({ type: "context", elements: [{ type: "mrkdwn", text: detail }] } unless detail.empty?)
+        ].compact
+      end
+    end
+  end
+end

data/lib/queue_pulse/notifiers/test.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module QueuePulse
+  module Notifiers
+    # Collects delivered alerts in memory. For use in tests and local dev.
+    class Test < Base
+      attr_reader :delivered
+
+      def initialize
+        super()
+        @delivered = []
+      end
+
+      def deliver(alerts)
+        @delivered.concat(alerts)
+      end
+
+      def clear
+        @delivered.clear
+      end
+    end
+  end
+end

data/lib/queue_pulse/notifiers/webhook.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module QueuePulse
+  module Notifiers
+    # Posts alerts as JSON to an arbitrary endpoint. Useful for piping into your
+    # own systems, PagerDuty Events API, Discord, etc.
+    class Webhook < Base
+      def initialize(url:, headers: {})
+        super()
+        @url = url
+        @headers = headers
+      end
+
+      def deliver(alerts)
+        return if alerts.empty?
+
+        post_json(@url, payload(alerts), headers: @headers)
+      end
+
+      def payload(alerts)
+        {
+          source: "queue_pulse",
+          version: QueuePulse::VERSION,
+          alerts: alerts.map(&:to_h)
+        }
+      end
+    end
+  end
+end

data/lib/queue_pulse/poller.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module QueuePulse
+  # Optional background poller. Runs QueuePulse.check! every poll_interval
+  # seconds in a daemon thread. Opt-in (call QueuePulse.start_poller!) because
+  # most apps prefer scheduling CheckJob via Solid Queue recurring tasks.
+  class Poller
+    def initialize(config)
+      @config = config
+      @thread = nil
+      @running = false
+    end
+
+    def start
+      return self if @running
+
+      @running = true
+      @thread = Thread.new do
+        Thread.current.name = "queue_pulse-poller"
+        loop do
+          break unless @running
+
+          QueuePulse.check!
+          sleep(@config.poll_interval)
+        end
+      end
+      @config.logger.info("[QueuePulse] poller started (every #{@config.poll_interval}s)")
+      self
+    end
+
+    def stop
+      @running = false
+      @thread&.wakeup
+      @thread = nil
+    end
+
+    def running?
+      @running
+    end
+  end
+
+  class << self
+    def start_poller!
+      @poller ||= Poller.new(config)
+      @poller.start
+    end
+
+    def stop_poller!
+      @poller&.stop
+    end
+  end
+end

data/lib/queue_pulse/railtie.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module QueuePulse
+  # Wires the gem into a Rails app: loads the rake tasks. The background poller
+  # is left opt-in (QueuePulse.start_poller!) to avoid surprising thread
+  # behaviour in web processes.
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path("../tasks/queue_pulse.rake", __dir__)
+    end
+  end
+end

data/lib/queue_pulse/source/solid_queue.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module QueuePulse
+  class Source
+    # Reads the real Solid Queue tables, read-only. All access goes through the
+    # solid_queue gem's own ActiveRecord models when present. Every method is
+    # defensive: if the schema isn't there yet, #available? returns false and
+    # the monitor skips all checks (Requirements US-2.4, US-7.5).
+    class SolidQueue < Source
+      def available?
+        return false unless defined?(::SolidQueue::Job)
+
+        ::SolidQueue::Job.connection.data_source_exists?("solid_queue_failed_executions")
+      rescue StandardError
+        false
+      end
+
+      def failed_executions_since(since_id, limit:)
+        ::SolidQueue::FailedExecution
+          .where("id > ?", since_id)
+          .order(:id)
+          .limit(limit)
+          .includes(:job)
+          .map do |fe|
+            job = fe.job
+            FailedRecord.new(
+              id: fe.id,
+              job_class: job&.class_name || "Unknown",
+              queue_name: job&.queue_name || "unknown",
+              error_message: extract_message(fe),
+              failed_at: fe.created_at
+            )
+          end
+      end
+
+      def max_failed_execution_id
+        ::SolidQueue::FailedExecution.maximum(:id)
+      end
+
+      def failed_count_since(since_id)
+        ::SolidQueue::FailedExecution.where("id > ?", since_id).count
+      end
+
+      def queue_stats
+        # Use ActiveRecord calculations (not a raw MIN pluck) so created_at comes
+        # back as a properly time-zone-cast Time, not a naive DB string. Parsing
+        # the string ourselves would misread UTC timestamps as local time and
+        # invent a timezone-sized latency (e.g. +9h in JST).
+        depths = ::SolidQueue::ReadyExecution.group(:queue_name).count
+        oldest = ::SolidQueue::ReadyExecution.group(:queue_name).minimum(:created_at)
+        depths.map do |queue_name, depth|
+          QueueStat.new(queue_name: queue_name, depth: depth.to_i, oldest_enqueued_at: oldest[queue_name])
+        end
+      end
+
+      def stuck_claimed_executions(older_than:)
+        cutoff = now - older_than
+        ::SolidQueue::ClaimedExecution
+          .where("created_at < ?", cutoff)
+          .includes(:job)
+          .map do |ce|
+            job = ce.job
+            ClaimedRecord.new(
+              job_id: ce.job_id,
+              job_class: job&.class_name || "Unknown",
+              queue_name: job&.queue_name || "unknown",
+              claimed_at: ce.created_at
+            )
+          end
+      end
+
+      def live_process_count(heartbeat_within:)
+        cutoff = now - heartbeat_within
+        ::SolidQueue::Process.where("last_heartbeat_at >= ?", cutoff).count
+      end
+
+      def total_process_count
+        ::SolidQueue::Process.count
+      end
+
+      private
+
+      def extract_message(failed_execution)
+        # Solid Queue stores a serialized error; prefer its parsed accessors,
+        # fall back to the first line of the raw error string.
+        if failed_execution.respond_to?(:exception_class) && failed_execution.exception_class
+          msg = failed_execution.respond_to?(:message) ? failed_execution.message : nil
+          return "#{failed_execution.exception_class}: #{msg}".strip[0, 500]
+        end
+
+        failed_execution.error.to_s.lines.first.to_s.strip[0, 500]
+      rescue StandardError
+        "(error details unavailable)"
+      end
+    end
+  end
+end

data/lib/queue_pulse/source.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module QueuePulse
+  # A Source abstracts every read of Solid Queue's tables. Production uses
+  # Source::SolidQueue; tests use Source::Fake. Checks depend only on this
+  # interface, so the core logic is verifiable without a real database.
+  class Source
+    # --- Value records returned to checks -------------------------------------
+    FailedRecord = Struct.new(:id, :job_class, :queue_name, :error_message, :failed_at, keyword_init: true)
+    QueueStat    = Struct.new(:queue_name, :depth, :oldest_enqueued_at, keyword_init: true)
+    ClaimedRecord = Struct.new(:job_id, :job_class, :queue_name, :claimed_at, keyword_init: true)
+
+    # Is the Solid Queue schema present and readable? When false, all checks
+    # are skipped (the app may simply not have Solid Queue set up yet).
+    def available?
+      raise NotImplementedError
+    end
+
+    def now
+      Time.now
+    end
+
+    # @return [Array<FailedRecord>] failed executions with id > since_id
+    def failed_executions_since(_since_id, limit:)
+      raise NotImplementedError
+    end
+
+    # @return [Integer, nil] the largest failed-execution id currently present
+    def max_failed_execution_id
+      raise NotImplementedError
+    end
+
+    # @return [Integer] number of failed executions with id > since_id
+    def failed_count_since(_since_id)
+      raise NotImplementedError
+    end
+
+    # @return [Array<QueueStat>] per-queue ready depth and oldest enqueue time
+    def queue_stats
+      raise NotImplementedError
+    end
+
+    # @return [Array<ClaimedRecord>] jobs claimed before (now - older_than secs)
+    def stuck_claimed_executions(older_than:)
+      raise NotImplementedError
+    end
+
+    # Number of processes whose heartbeat is within `heartbeat_within` seconds.
+    def live_process_count(heartbeat_within:)
+      raise NotImplementedError
+    end
+
+    # Total processes ever registered (used to distinguish "never started" from
+    # "all died").
+    def total_process_count
+      raise NotImplementedError
+    end
+
+    # In-memory implementation for tests and local experiments.
+    class Fake < Source
+      attr_accessor :available, :clock
+
+      def initialize(now: Time.now)
+        super()
+        @available = true
+        @clock = now
+        @failed = []
+        @queue_stats = []
+        @claimed = []
+        @processes = [] # array of heartbeat Times
+      end
+
+      def available? = @available
+      def now = @clock
+
+      # --- test setters --------------------------------------------------------
+      def add_failed(id:, job_class:, queue_name:, error_message:, failed_at: @clock)
+        @failed << FailedRecord.new(id: id, job_class: job_class, queue_name: queue_name,
+                                    error_message: error_message, failed_at: failed_at)
+      end
+
+      def set_queue(queue_name, depth:, oldest_enqueued_at: nil)
+        @queue_stats.reject! { |q| q.queue_name == queue_name }
+        @queue_stats << QueueStat.new(queue_name: queue_name, depth: depth, oldest_enqueued_at: oldest_enqueued_at)
+      end
+
+      def add_claimed(job_id:, job_class:, queue_name:, claimed_at:)
+        @claimed << ClaimedRecord.new(job_id: job_id, job_class: job_class, queue_name: queue_name, claimed_at: claimed_at)
+      end
+
+      def add_process(last_heartbeat_at:)
+        @processes << last_heartbeat_at
+      end
+
+      # --- Source interface ----------------------------------------------------
+      def failed_executions_since(since_id, limit:)
+        @failed.select { |f| f.id > since_id }.sort_by(&:id).first(limit)
+      end
+
+      def max_failed_execution_id
+        @failed.map(&:id).max
+      end
+
+      def failed_count_since(since_id)
+        @failed.count { |f| f.id > since_id }
+      end
+
+      def queue_stats
+        @queue_stats
+      end
+
+      def stuck_claimed_executions(older_than:)
+        cutoff = @clock - older_than
+        @claimed.select { |c| c.claimed_at < cutoff }
+      end
+
+      def live_process_count(heartbeat_within:)
+        cutoff = @clock - heartbeat_within
+        @processes.count { |hb| hb >= cutoff }
+      end
+
+      def total_process_count
+        @processes.size
+      end
+    end
+  end
+end

data/lib/queue_pulse/state.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module QueuePulse
+  # Tiny wrapper over the configured cache for the two pieces of state QueuePulse
+  # needs: the failed-execution high-water mark, and per-fingerprint cooldowns.
+  #
+  # Backed by Rails.cache (or MemoryStore fallback). If the cache evicts our
+  # keys we fail safe: at worst a duplicate alert, never a crash. (Requirements
+  # US-2.2, US-3.3, non-functional #3.)
+  class State
+    NAMESPACE = "queue_pulse"
+    HWM_KEY = "#{NAMESPACE}:failed_high_water_mark"
+
+    def initialize(cache)
+      @cache = cache
+    end
+
+    def failed_high_water_mark
+      value = @cache.read(HWM_KEY)
+      value && Integer(value)
+    rescue ArgumentError, TypeError
+      nil
+    end
+
+    def set_failed_high_water_mark(id)
+      @cache.write(HWM_KEY, id.to_i)
+    end
+
+    # True if this fingerprint was alerted within its cooldown window.
+    def alerted_recently?(fingerprint)
+      !@cache.read(cooldown_key(fingerprint)).nil?
+    end
+
+    def mark_alerted(fingerprint, ttl)
+      @cache.write(cooldown_key(fingerprint), 1, expires_in: ttl)
+    end
+
+    private
+
+    def cooldown_key(fingerprint)
+      "#{NAMESPACE}:cooldown:#{fingerprint}"
+    end
+  end
+end

data/lib/queue_pulse/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module QueuePulse
+  VERSION = "0.1.0"
+end