sidekiq-vigil 0.1.0

data/lib/sidekiq_vigil/check/queue_size.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class QueueSize < Base
+      def call
+        api.queues(options.fetch(:queues, :all)).map do |queue|
+          thresholds = options.merge(options.fetch(:per_queue, {}).fetch(queue.name, {}))
+          threshold_result(
+            target: queue.name,
+            value: queue.size,
+            warn: thresholds.fetch(:warn, 1_000),
+            critical: thresholds.fetch(:critical, 10_000),
+            message: "#{queue.size} jobs waiting"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/redis_health.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class RedisHealth < Base
+      def initialize(
+        storage:,
+        options: {},
+        clock: -> { Time.now },
+        logger: SidekiqVigil.logger,
+        api: SidekiqApi.new,
+        monotonic_clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      )
+        super(storage:, options:, clock:, logger:, api:)
+        @monotonic_clock = monotonic_clock
+      end
+
+      def call
+        started_at = monotonic_clock.call
+        storage.ping
+        latency_ms = (monotonic_clock.call - started_at) * 1_000
+        [latency_result(latency_ms), memory_result]
+      end
+
+      private
+
+      attr_reader :monotonic_clock
+
+      def latency_result(latency_ms)
+        threshold_result(
+          target: "latency",
+          value: latency_ms,
+          warn: options.fetch(:latency_ms, 100),
+          critical: options.fetch(:critical_latency_ms, options.fetch(:latency_ms, 100) * 2),
+          message: format("%.2fms PING latency", latency_ms)
+        )
+      end
+
+      def memory_result
+        info = storage.info("memory")
+        used = info.fetch("used_memory", 0).to_f
+        maximum = info.fetch("maxmemory", 0).to_f
+        ratio = maximum.positive? ? used / maximum : 0.0
+        threshold_result(
+          target: "memory",
+          value: ratio,
+          warn: options.fetch(:memory_pct, 0.9),
+          critical: options.fetch(:critical_memory_pct, 0.98),
+          message: maximum.positive? ? format("%.1f%% Redis maxmemory used", ratio * 100) : "Redis maxmemory is unlimited",
+          metadata: { used_memory: used.to_i, maxmemory: maximum.to_i }
+        )
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/registry.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class Registry
+      def initialize
+        @checks = {}
+      end
+
+      def register(name, klass)
+        @checks[name.to_sym] = klass
+      end
+
+      def fetch(name)
+        @checks.fetch(name.to_sym)
+      end
+
+      def names
+        @checks.keys
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/scheduled_backlog.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class ScheduledBacklog < Base
+      def call
+        overdue_count = api.scheduled.count do |job|
+          timestamp = job.respond_to?(:at) ? job.at : Time.at(job.score.to_f)
+          timestamp <= clock.call - options.fetch(:overdue, 300)
+        end
+        threshold_result(
+          target: "global",
+          value: overdue_count,
+          warn: options.fetch(:warn, 1),
+          critical: options.fetch(:critical, 100),
+          message: "#{overdue_count} overdue scheduled jobs"
+        )
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/set_size.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class SetSize < Base
+      STATE_TTL = 8 * 24 * 60 * 60
+
+      def call
+        total = current_size
+        value = growth_value(total)
+        threshold_result(
+          target: "global",
+          value:,
+          warn: options.fetch(:warn, default_warn),
+          critical: options.fetch(:critical, default_critical),
+          message: message(total, value),
+          metadata: { total: }
+        )
+      end
+
+      private
+
+      def growth_value(total)
+        return total unless options.fetch(:growth_only, false)
+
+        previous = storage.get(state_key)
+        storage.set(state_key, total, ttl: STATE_TTL)
+        return 0 unless previous
+
+        [total - previous.to_i, 0].max
+      end
+
+      def state_key
+        "check_state:#{check_name}"
+      end
+
+      def message(total, value)
+        return "#{total} entries" unless options.fetch(:growth_only, false)
+
+        "#{value} new entries (#{total} total)"
+      end
+    end
+
+    class RetrySet < SetSize
+      private
+
+      def current_size
+        api.retry_size
+      end
+
+      def default_warn
+        100
+      end
+
+      def default_critical
+        1_000
+      end
+    end
+
+    class DeadSet < SetSize
+      private
+
+      def current_size
+        api.dead_size
+      end
+
+      def default_warn
+        1
+      end
+
+      def default_critical
+        50
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/stuck_jobs.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class StuckJobs < Base
+      def call
+        stuck = api.workers.filter_map { |worker| stuck_result(worker) }
+        return stuck unless stuck.empty?
+
+        [Result.new(check_name:, severity: :ok, value: 0, threshold: threshold, message: "no stuck jobs")]
+      end
+
+      private
+
+      def stuck_result(worker)
+        payload = worker.fetch(:work)
+        started_at = payload["run_at"] || payload[:run_at]
+        return unless started_at
+
+        age = clock.call.to_f - started_at.to_f
+        return if age < threshold
+
+        Result.new(
+          check_name:,
+          target: "#{worker[:process_id]}:#{worker[:thread_id]}",
+          severity: :critical,
+          value: age,
+          threshold:,
+          message: "job has run for #{age.round}s",
+          metadata: { payload: safe_payload(payload) }
+        )
+      end
+
+      def safe_payload(payload)
+        raw = payload["payload"] || payload[:payload] || {}
+        { class: raw["class"] || raw[:class], queue: raw["queue"] || raw[:queue] }
+      end
+
+      def threshold
+        options.fetch(:threshold, 1_800)
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/throughput_anomaly.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class ThroughputAnomaly < Base
+      WINDOW_MINUTES = 60
+
+      def call
+        baseline = baseline_samples
+        return insufficient_result(baseline.length) if baseline.length < options.fetch(:min_samples, 3)
+
+        expected = median(baseline)
+        current = window_total(clock.call)
+        drop = expected.positive? ? 1 - current.fdiv(expected) : 0.0
+        severity = drop >= options.fetch(:drop_pct, 0.5) ? :warn : :ok
+        Result.new(
+          check_name:,
+          severity:,
+          value: current,
+          threshold: expected * (1 - options.fetch(:drop_pct, 0.5)),
+          message: format(
+            "%<current>d jobs vs %<expected>.1f baseline (%<drop>.1f%% drop)",
+            current:,
+            expected:,
+            drop: drop * 100
+          ),
+          metadata: { baseline: expected, samples: baseline.length, drop_pct: drop }
+        )
+      end
+
+      private
+
+      def baseline_samples
+        days = options.fetch(:baseline_days, 7)
+        (1..days).filter_map do |days_ago|
+          sample_time = Timezone.same_local_days_ago(clock.call, days_ago, options.fetch(:timezone, "UTC"))
+          sample = window_total(sample_time)
+          sample if sample.positive?
+        end
+      end
+
+      def window_total(time)
+        local = Timezone.local_time(time, options.fetch(:timezone, "UTC"))
+        WINDOW_MINUTES.times.sum do |offset|
+          minute = local - (offset * 60)
+          storage.hash_get_all("stats:#{minute.getutc.strftime('%Y%m%d%H%M')}").fetch("processed", 0).to_i
+        end
+      end
+
+      def median(values)
+        sorted = values.sort
+        middle = sorted.length / 2
+        return sorted[middle].to_f if sorted.length.odd?
+
+        (sorted[middle - 1] + sorted[middle]).fdiv(2)
+      end
+
+      def insufficient_result(samples)
+        Result.new(
+          check_name:,
+          severity: :ok,
+          value: nil,
+          message: "insufficient baseline samples (#{samples}/#{options.fetch(:min_samples, 3)})",
+          metadata: { insufficient_samples: true, samples: }
+        )
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/utilization.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class Utilization < Base
+      STATE_TTL_PADDING = 300
+
+      def call
+        busy, concurrency = totals
+        ratio = concurrency.zero? ? 0.0 : busy.fdiv(concurrency)
+        severity, threshold = sustained_severity(ratio)
+        Result.new(
+          check_name:,
+          severity:,
+          value: ratio,
+          threshold:,
+          message: format(
+            "%<ratio>.1f%% utilized (%<busy>d/%<concurrency>d)",
+            ratio: ratio * 100,
+            busy:,
+            concurrency:
+          )
+        )
+      end
+
+      private
+
+      def totals
+        api.processes.each_with_object([0, 0]) do |process, totals|
+          totals[0] += process_value(process, "busy").to_i
+          totals[1] += process_value(process, "concurrency").to_i
+        end
+      end
+
+      def process_value(process, key)
+        process[key] || process[key.to_sym]
+      end
+
+      def sustained_severity(ratio)
+        warn = options.fetch(:warn, 0.85)
+        critical = options.fetch(:critical, 0.95)
+        return clear_sustained_state(warn) if ratio < warn
+
+        started_at = storage.get("check_state:utilization_since")
+        storage.set("check_state:utilization_since", started_at || clock.call.to_f, ttl: state_ttl)
+        return [:ok, warn] if started_at.nil? || clock.call.to_f - started_at.to_f < options.fetch(:sustained, 300)
+
+        severity_and_threshold(ratio, warn, critical)
+      end
+
+      def clear_sustained_state(warn)
+        storage.delete("check_state:utilization_since")
+        [:ok, warn]
+      end
+
+      def state_ttl
+        options.fetch(:sustained, 300) + STATE_TTL_PADDING
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/checker.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module SidekiqVigil
+  class Checker
+    attr_reader :thread
+
+    def initialize(
+      storage:,
+      config:,
+      leader_election: nil,
+      alert_manager: nil,
+      notifier_manager: nil,
+      logger: SidekiqVigil.logger,
+      clock: -> { Time.now },
+      sleeper: ->(seconds) { sleep(seconds) }
+    )
+      @storage = storage
+      @config = config
+      @logger = logger
+      @clock = clock
+      @sleeper = sleeper
+      @leader_election = leader_election || LeaderElection.new(storage:, ttl: config.interval * 3)
+      @notifier_manager = notifier_manager || build_notifier_manager
+      @alert_manager = alert_manager || build_alert_manager
+      @running = false
+      @direct_redis_notified = false
+    end
+
+    def start
+      return thread if thread&.alive?
+
+      @running = true
+      @thread = Thread.new { run }
+      @thread.name = "sidekiq-vigil-checker" if @thread.respond_to?(:name=)
+      @thread
+    end
+
+    def stop
+      @running = false
+      thread&.wakeup if thread&.status == "sleep"
+      thread&.join(config.interval + 1)
+      leader_election.release
+      @thread = nil
+    end
+
+    def run_once
+      return false unless become_or_remain_leader
+
+      results = configured_checks.flat_map(&:execute)
+      events = alert_manager.process(results)
+      write_snapshot(results)
+      warn_config_drift
+      notifier_manager.notify(events)
+      @direct_redis_notified = false
+      results
+    rescue StandardError => e
+      logger.error("[sidekiq-vigil] checker cycle failed: #{e.class}: #{e.message}")
+      notify_redis_outage(e) if redis_error?(e)
+      false
+    end
+
+    private
+
+    attr_reader :storage, :config, :leader_election, :alert_manager, :notifier_manager, :logger, :clock, :sleeper
+
+    def run
+      while @running
+        run_once
+        sleeper.call(config.interval)
+      end
+    rescue StandardError => e
+      logger.error("[sidekiq-vigil] checker thread failed: #{e.class}: #{e.message}")
+    ensure
+      leader_election.release
+    end
+
+    def become_or_remain_leader
+      return leader_election.extend if leader_election.leader?
+
+      leader_election.acquire
+    end
+
+    def configured_checks
+      config.checks.map do |definition|
+        klass = definition.klass || BUILT_IN_CHECKS.fetch(definition.name)
+        options = definition.options.merge(timezone: config.timezone)
+        klass.new(storage:, options:, clock:, logger:)
+      end
+    end
+
+    def write_snapshot(results)
+      payload = JSON.generate(
+        timestamp: clock.call.utc.iso8601,
+        results: results.map(&:to_h),
+        alerts: alert_states
+      )
+      storage.set("snapshot", payload, ttl: config.interval * 4)
+    end
+
+    def alert_states
+      storage.hash_get_all("alerts").transform_values { |payload| JSON.parse(payload) }
+    end
+
+    def warn_config_drift
+      digests = storage.hash_get_all("config_digest")
+      return if digests.values.uniq.length <= 1
+
+      logger.warn("[sidekiq-vigil] configuration digest mismatch across processes")
+    end
+
+    def build_notifier_manager
+      Notifier::Manager.new(notifiers: Notifier::Factory.build(config, logger:), logger:)
+    end
+
+    def build_alert_manager
+      mute = Alert::Mute.new(
+        storage:,
+        schedules: config.alerting.mutes,
+        clock:,
+        timezone: config.timezone
+      )
+      Alert::Manager.new(storage:, config: config.alerting, mute:, clock:)
+    end
+
+    def notify_redis_outage(error)
+      return if @direct_redis_notified
+
+      @direct_redis_notified = true
+      result = Result.new(
+        check_name: "redis_health",
+        target: "connection",
+        severity: :error,
+        message: "Redis unavailable (#{error.class})"
+      )
+      state = Alert::State.new(status: "firing", severity: "error", message: result.message)
+      event = Alert::Event.new(result:, transition: :firing, state:, history: [], timestamp: clock.call)
+      notifier_manager.notify([event])
+    end
+
+    def redis_error?(error)
+      return true if defined?(RedisClient::Error) && error.is_a?(RedisClient::Error)
+
+      error.message.match?(/redis|connection|socket|timeout/i)
+    end
+  end
+end