sidekiq-vigil 0.1.0

data/lib/sidekiq_vigil/cli.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require "json"
+require "optparse"
+
+module SidekiqVigil
+  class CLI
+    EXIT_OK = 0
+    EXIT_ALERT = 2
+    EXIT_UNAVAILABLE = 3
+
+    def initialize(config: SidekiqVigil.config, storage: nil, out: $stdout, err: $stderr, checker_factory: nil)
+      @config = config
+      @resolved_storage = storage
+      @out = out
+      @err = err
+      @checker_factory = checker_factory || -> { Checker.new(storage: resolved_storage, config:) }
+    end
+
+    def run(arguments)
+      args = arguments.dup
+      load_config(extract_config_path(args))
+      configure_sidekiq_api
+      command = args.shift || "run"
+      handler = command_handlers(args)[command]
+      return unknown_command(command) unless handler
+
+      handler.call
+    rescue OptionParser::ParseError, ConfigError, ArgumentError => e
+      err.puts(e.message)
+      EXIT_UNAVAILABLE
+    end
+
+    private
+
+    attr_reader :config, :out, :err, :checker_factory
+
+    def resolved_storage
+      @resolved_storage ||= SidekiqVigil.build_storage(config)
+    end
+
+    def command_handlers(args)
+      {
+        "run" => -> { run_forever(args) },
+        "check" => -> { run_check(args) },
+        "status" => -> { print_status(args) },
+        "test-notify" => -> { test_notify(args) },
+        "mute" => -> { mute(args) },
+        "unmute" => -> { unmute(args) }
+      }
+    end
+
+    def extract_config_path(args)
+      index = args.index("--config") || args.index("-c")
+      return unless index
+
+      option = args.delete_at(index)
+      path = args.delete_at(index)
+      raise OptionParser::MissingArgument, option unless path
+
+      path
+    end
+
+    def load_config(path)
+      return unless path
+      raise ConfigError, "config file not found: #{path}" unless File.file?(path)
+
+      Kernel.load(File.expand_path(path))
+      config.validate!
+    end
+
+    def configure_sidekiq_api
+      return unless config.redis
+
+      options = config.redis.compact.reject { |key, _value| %i[pool_size pool_timeout].include?(key.to_sym) }
+      Sidekiq.configure_client { |sidekiq| sidekiq.redis = options }
+    end
+
+    def run_forever(_args = [])
+      checker = checker_factory.call
+      reader, writer = IO.pipe
+      previous = install_signal_handlers(writer)
+      checker.start
+      reader.read(1)
+      checker.stop
+      EXIT_OK
+    ensure
+      restore_signal_handlers(previous) if previous
+      reader&.close
+      writer&.close
+    end
+
+    def install_signal_handlers(writer)
+      %w[INT TERM].to_h do |signal|
+        previous = Signal.trap(signal) { writer.write_nonblock(signal, exception: false) }
+        [signal, previous]
+      end
+    end
+
+    def restore_signal_handlers(previous)
+      previous.each { |signal, handler| Signal.trap(signal, handler) }
+    end
+
+    def run_check(args)
+      parser = OptionParser.new
+      once = false
+      parser.on("--once") { once = true }
+      parser.parse!(args)
+      raise OptionParser::InvalidOption, "check requires --once" unless once
+
+      results = checker_factory.call.run_once
+      return EXIT_UNAVAILABLE if results == false
+
+      print_results(results)
+      results.all?(&:ok?) ? EXIT_OK : EXIT_ALERT
+    end
+
+    def print_results(results)
+      results.each do |result|
+        out.puts([result.severity.to_s.upcase, result.check_name, result.target, result.message].compact.join("\t"))
+      end
+    end
+
+    def print_status(_args = [])
+      raw = resolved_storage.get("snapshot")
+      return unavailable("No snapshot available") unless raw
+
+      snapshot = JSON.parse(raw)
+      out.puts("Snapshot: #{snapshot.fetch('timestamp')}")
+      snapshot.fetch("results").each do |result|
+        alert_id = "#{result.fetch('check_name')}:#{result.fetch('target')}"
+        alert_status = snapshot.fetch("alerts", {}).dig(alert_id, "status") || "ok"
+        out.puts(
+          [result.fetch("severity").upcase, result.fetch("check_name"), result.fetch("target"), alert_status.upcase].join("\t")
+        )
+      end
+      EXIT_OK
+    end
+
+    def test_notify(_args = [])
+      result = Result.new(
+        check_name: "test_notification",
+        severity: :warn,
+        message: "Sidekiq Vigil test notification"
+      )
+      state = Alert::State.new(status: "firing", severity: "warn")
+      event = Alert::Event.new(result:, transition: :firing, state:, history: [], timestamp: Time.now)
+      manager = Notifier::Manager.new(notifiers: Notifier::Factory.build(config))
+      manager.notify([event])
+      out.puts("Test notification sent to #{config.runnable_notifiers.length} notifier(s)")
+      EXIT_OK
+    end
+
+    def mute(args)
+      reason = nil
+      parser = OptionParser.new
+      parser.on("--reason REASON") { |value| reason = value }
+      parser.parse!(args)
+      duration = parse_duration(args.shift)
+      raise OptionParser::InvalidArgument, "mute duration is required" unless duration
+
+      alert_mute.mute(duration, reason:)
+      out.puts("Muted for #{duration} seconds")
+      EXIT_OK
+    end
+
+    def unmute(_args = [])
+      alert_mute.unmute
+      out.puts("Mute cleared")
+      EXIT_OK
+    end
+
+    def alert_mute
+      Alert::Mute.new(
+        storage: resolved_storage,
+        schedules: config.alerting.mutes,
+        timezone: config.timezone
+      )
+    end
+
+    def parse_duration(value)
+      return unless value
+
+      match = value.match(/\A(\d+)([smhd]?)\z/)
+      raise OptionParser::InvalidArgument, "invalid duration: #{value}" unless match
+
+      Integer(match[1]) * { "" => 1, "s" => 1, "m" => 60, "h" => 3_600, "d" => 86_400 }.fetch(match[2])
+    end
+
+    def unavailable(message)
+      err.puts(message)
+      EXIT_UNAVAILABLE
+    end
+
+    def unknown_command(command)
+      err.puts("Unknown command: #{command}")
+      err.puts(usage)
+      EXIT_UNAVAILABLE
+    end
+
+    def usage
+      "Usage: vigil [--config FILE] [run|check --once|status|test-notify|mute DURATION|unmute]"
+    end
+  end
+end

data/lib/sidekiq_vigil/collector.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  class Collector
+    Snapshot = Data.define(:stats, :executions)
+
+    def initialize(clock: -> { Time.now })
+      @clock = clock
+      @mutex = Mutex.new
+      @stats = empty_stats
+      @executions = empty_executions
+    end
+
+    def record(worker:, queue:, duration:, failed:)
+      minute = clock.call.utc.strftime("%Y%m%d%H%M")
+      worker_name = worker.respond_to?(:name) ? worker.name : worker.class.name
+      @mutex.synchronize do
+        update_stats(minute, worker_name, failed)
+        update_execution(queue.to_s, duration)
+      end
+    end
+
+    def drain
+      @mutex.synchronize do
+        snapshot = Snapshot.new(stats: @stats, executions: @executions)
+        @stats = empty_stats
+        @executions = empty_executions
+        snapshot
+      end
+    end
+
+    def merge(snapshot)
+      @mutex.synchronize do
+        snapshot.stats.each do |minute, fields|
+          fields.each { |field, value| @stats[minute][field] += value }
+        end
+        snapshot.executions.each do |queue, values|
+          current = @executions[queue]
+          current[:count] += values[:count]
+          current[:sum] += values[:sum]
+          current[:max] = [current[:max], values[:max]].max
+        end
+      end
+    end
+
+    private
+
+    attr_reader :clock
+
+    def empty_stats
+      Hash.new { |hash, key| hash[key] = Hash.new(0) }
+    end
+
+    def empty_executions
+      Hash.new { |hash, key| hash[key] = { count: 0, sum: 0.0, max: 0.0 } }
+    end
+
+    def update_stats(minute, worker_name, failed)
+      @stats[minute]["processed"] += 1
+      @stats[minute]["processed:#{worker_name}"] += 1
+      return unless failed
+
+      @stats[minute]["failed"] += 1
+      @stats[minute]["failed:#{worker_name}"] += 1
+    end
+
+    def update_execution(queue, duration)
+      execution = @executions[queue]
+      execution[:count] += 1
+      execution[:sum] += duration
+      execution[:max] = [execution[:max], duration].max
+    end
+  end
+end

data/lib/sidekiq_vigil/config.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+
+module SidekiqVigil
+  class ConfigError < Error; end
+
+  class Config
+    CheckDefinition = Data.define(:name, :klass, :options)
+    class NotifierDefinition
+      SECRET_KEYS = %i[webhook_url routes bot_token url].freeze
+
+      attr_reader :name, :klass, :options
+
+      def initialize(name:, klass:, options:)
+        @name = name
+        @klass = klass
+        @options = options
+      end
+
+      def inspect
+        safe_options = options.transform_values.with_index do |value, _index|
+          value.is_a?(Hash) ? value.transform_values { "[FILTERED]" } : value
+        end
+        SECRET_KEYS.each { |key| safe_options[key] = "[FILTERED]" if safe_options.key?(key) }
+        "#<#{self.class} name=#{name.inspect} options=#{safe_options.inspect}>"
+      end
+    end
+
+    BUILT_IN_CHECKS = %i[
+      queue_latency queue_size retry_set dead_set process_alive utilization failure_rate stuck_jobs memory redis_health
+      scheduled_backlog throughput_anomaly
+    ].freeze
+    DEFAULT_CHECKS = (BUILT_IN_CHECKS - [:throughput_anomaly]).freeze
+    SLACK_MENTION = /\A(?:<@U[A-Z0-9]+>|<!subteam\^S[A-Z0-9]+>|<!here>)\z/
+
+    attr_accessor :interval, :flush_interval, :key_prefix, :timezone, :redis
+    attr_reader :checks, :notifiers
+
+    def initialize(environment: nil)
+      @environment = environment || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      @enabled = true
+      @enabled_explicit = false
+      @interval = 30
+      @flush_interval = 10
+      @key_prefix = "default"
+      @timezone = "UTC"
+      @redis = nil
+      @checks = DEFAULT_CHECKS.map { |name| CheckDefinition.new(name:, klass: nil, options: {}) }
+      @notifiers = []
+      @alerting = AlertingConfig.new
+    end
+
+    def enabled=(value)
+      @enabled_explicit = true
+      @enabled = value
+    end
+
+    def enabled?
+      @enabled == true
+    end
+
+    def production?
+      environment == "production"
+    end
+
+    def external_notifications_enabled?
+      enabled? && (production? || @enabled_explicit)
+    end
+
+    def check(name_or_class, **options)
+      name, klass = normalize_check(name_or_class)
+      checks.reject! { |definition| definition.name == name }
+      checks << CheckDefinition.new(name:, klass:, options:)
+    end
+
+    def disable_check(name)
+      checks.reject! { |definition| definition.name == name.to_sym }
+    end
+
+    def notifier(name_or_class, **options)
+      name, klass = normalize_notifier(name_or_class)
+      notifiers << NotifierDefinition.new(name:, klass:, options:)
+    end
+
+    def alerting
+      yield @alerting if block_given?
+      @alerting
+    end
+
+    def validate!
+      validate_positive!(:interval, interval)
+      validate_positive!(:flush_interval, flush_interval)
+      raise ConfigError, "key_prefix must not be empty" if key_prefix.to_s.empty?
+      raise ConfigError, "key_prefix contains unsupported characters" unless key_prefix.to_s.match?(/\A[\w.-]+\z/)
+      raise ConfigError, "timezone must not be empty" if timezone.to_s.empty?
+
+      validate_redis!
+      checks.each { |definition| validate_options!(definition.options, "check #{definition.name}") }
+      notifiers.each { |definition| validate_notifier!(definition) }
+      alerting.validate!
+      self
+    end
+
+    def digest
+      payload = {
+        enabled: enabled?,
+        interval: interval,
+        flush_interval: flush_interval,
+        key_prefix: key_prefix,
+        timezone: timezone,
+        checks: checks.map { |item| [item.name, item.options] },
+        notifiers: notifiers.map { |item| [item.name, item.options] },
+        alerting: alerting.to_h
+      }
+      Digest::SHA256.hexdigest(JSON.generate(payload))
+    end
+
+    def runnable_notifiers
+      validate!
+      definitions = notifiers.empty? ? [NotifierDefinition.new(name: :log, klass: nil, options: {})] : notifiers
+      return definitions if external_notifications_enabled?
+
+      definitions.select { |definition| definition.name == :log }
+    end
+
+    private
+
+    attr_reader :environment
+
+    def normalize_check(name_or_class)
+      if name_or_class.is_a?(Class)
+        return [name_or_class.name.split("::").last.gsub(/([a-z])([A-Z])/, '\1_\2').downcase.to_sym,
+                name_or_class]
+      end
+
+      name = name_or_class.to_sym
+      raise ConfigError, "unknown check: #{name}" unless BUILT_IN_CHECKS.include?(name)
+
+      [name, nil]
+    end
+
+    def normalize_notifier(name_or_class)
+      return [name_or_class.name.split("::").last.downcase.to_sym, name_or_class] if name_or_class.is_a?(Class)
+
+      name = name_or_class.to_sym
+      raise ConfigError, "unknown notifier: #{name}" unless %i[log slack webhook].include?(name)
+
+      [name, nil]
+    end
+
+    def validate_positive!(name, value)
+      raise ConfigError, "#{name} must be positive" unless value.is_a?(Numeric) && value.positive?
+    end
+
+    def validate_options!(options, context)
+      options.each { |key, value| validate_option!(key, value, context) }
+    end
+
+    def validate_option!(key, value, context)
+      return value.each_value { |thresholds| validate_options!(thresholds, context) } if key.to_sym == :per_queue && value.is_a?(Hash)
+      return unless numeric_threshold?(key)
+      return if value.is_a?(Numeric) && !value.negative?
+
+      raise ConfigError, "#{context} #{key} must not be negative"
+    end
+
+    def numeric_threshold?(key)
+      key.to_s.match?(/(?:warn|critical|threshold|overdue|window|samples|processes|sustained|drop_pct|baseline_days|latency_ms|memory_pct)/)
+    end
+
+    def validate_notifier!(definition)
+      validate_options!(definition.options, "notifier #{definition.name}")
+      return unless definition.name == :slack
+
+      mentions = definition.options.fetch(:mention, {})
+      mentions.each_value do |mention|
+        next if mention.nil? || mention.match?(SLACK_MENTION)
+
+        raise ConfigError,
+              "invalid Slack mention #{mention.inspect}; use <@U…>, <!subteam^S…>, or <!here>"
+      end
+    end
+
+    def validate_redis!
+      return unless redis
+      raise ConfigError, "redis must be a Hash" unless redis.is_a?(Hash)
+
+      pool_size = redis_option(:pool_size, 5)
+      pool_timeout = redis_option(:pool_timeout, 1)
+      raise ConfigError, "redis pool_size must be a positive integer" unless pool_size.is_a?(Integer) && pool_size.positive?
+      return if pool_timeout.is_a?(Numeric) && pool_timeout.positive?
+
+      raise ConfigError, "redis pool_timeout must be positive"
+    end
+
+    def redis_option(name, default)
+      redis.fetch(name) { redis.fetch(name.to_s, default) }
+    end
+  end
+
+  class AlertingConfig
+    attr_accessor :pending_cycles, :cooldown, :resolve_notice, :flap_window, :flap_threshold, :escalate_after,
+                  :group_threshold, :group_top_n, :mutes
+
+    def initialize
+      @pending_cycles = 2
+      @cooldown = 600
+      @resolve_notice = true
+      @flap_window = 120
+      @flap_threshold = 4
+      @escalate_after = nil
+      @group_threshold = 5
+      @group_top_n = 5
+      @mutes = []
+    end
+
+    def validate!
+      validate_numeric_fields!
+      validate_escalation!
+      mutes.each { |mute| validate_mute!(mute) }
+      self
+    end
+
+    def to_h
+      {
+        pending_cycles: pending_cycles,
+        cooldown: cooldown,
+        resolve_notice: resolve_notice,
+        flap_window: flap_window,
+        flap_threshold: flap_threshold,
+        escalate_after: escalate_after,
+        group_threshold: group_threshold,
+        group_top_n: group_top_n,
+        mutes: mutes
+      }
+    end
+
+    private
+
+    def validate_numeric_fields!
+      %i[cooldown flap_window group_threshold].each do |name|
+        value = public_send(name)
+        raise ConfigError, "#{name} must not be negative" unless value.is_a?(Numeric) && !value.negative?
+      end
+      %i[pending_cycles flap_threshold group_top_n].each do |name|
+        value = public_send(name)
+        raise ConfigError, "#{name} must be a positive integer" unless value.is_a?(Integer) && value.positive?
+      end
+    end
+
+    def validate_escalation!
+      return unless escalate_after
+      return if escalate_after.is_a?(Numeric) && !escalate_after.negative?
+
+      raise ConfigError, "escalate_after must not be negative"
+    end
+
+    def validate_mute!(mute)
+      Alert::Cron.new(mute.fetch(:cron))
+
+      duration = mute.fetch(:duration)
+      raise ConfigError, "mute duration must be positive" unless duration.is_a?(Numeric) && duration.positive?
+    end
+  end
+end

data/lib/sidekiq_vigil/health_app.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module SidekiqVigil
+  class HealthApp
+    JSON_HEADERS = { "content-type" => "application/json", "cache-control" => "no-store" }.freeze
+
+    def initialize(storage:, interval:, clock: -> { Time.now })
+      @storage = storage
+      @interval = interval
+      @clock = clock
+    end
+
+    def call(env)
+      return response(405, error: "method not allowed") unless env["REQUEST_METHOD"] == "GET"
+
+      case env["PATH_INFO"]
+      when "/healthz" then health
+      when "/status.json" then status
+      else response(404, error: "not found")
+      end
+    rescue StandardError => e
+      response(503, healthy: false, error: "snapshot unavailable", detail: e.class.name)
+    end
+
+    private
+
+    attr_reader :storage, :interval, :clock
+
+    def health
+      report = snapshot_report
+      response(report[:healthy] ? 200 : 503, report.slice(:healthy, :fresh, :age_seconds))
+    end
+
+    def status
+      report = snapshot_report
+      response(report[:healthy] ? 200 : 503, report)
+    end
+
+    def snapshot_report
+      raw = storage.get("snapshot")
+      return { healthy: false, fresh: false, age_seconds: nil, error: "snapshot missing" } unless raw
+
+      snapshot = JSON.parse(raw)
+      age = clock.call - Time.iso8601(snapshot.fetch("timestamp"))
+      fresh = age <= interval * 2
+      alerts = snapshot.fetch("alerts", {})
+      results = snapshot.fetch("results").map { |result| with_alert_state(result, alerts) }
+      checks_ok = results.all? { |result| result.fetch("severity") == "ok" }
+      snapshot.merge("results" => results, "alerts" => alerts, healthy: checks_ok && fresh, fresh:, age_seconds: age.round(3))
+    end
+
+    def with_alert_state(result, alerts)
+      alert_id = "#{result.fetch('check_name')}:#{result.fetch('target')}"
+      result.merge("alert_state" => alerts.fetch(alert_id, { "status" => "ok" }))
+    end
+
+    def response(status, body)
+      [status, JSON_HEADERS, [JSON.generate(body)]]
+    end
+  end
+end

data/lib/sidekiq_vigil/leader_election.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module SidekiqVigil
+  class LeaderElection
+    EXTEND_SCRIPT = <<~LUA
+      if redis.call("get", KEYS[1]) == ARGV[1] then
+        return redis.call("pexpire", KEYS[1], ARGV[2])
+      end
+      return 0
+    LUA
+    RELEASE_SCRIPT = <<~LUA
+      if redis.call("get", KEYS[1]) == ARGV[1] then
+        return redis.call("del", KEYS[1])
+      end
+      return 0
+    LUA
+
+    attr_reader :token
+
+    def initialize(storage:, ttl:, token: SecureRandom.uuid)
+      @storage = storage
+      @ttl_ms = (ttl * 1000).to_i
+      @token = token
+      @leader = false
+    end
+
+    def acquire
+      @leader = storage.acquire_lock("leader", token, ttl_ms: ttl_ms)
+    end
+
+    def extend
+      @leader = storage.eval(EXTEND_SCRIPT, keys: ["leader"], argv: [token, ttl_ms]).to_i == 1
+    end
+
+    def leader?
+      @leader
+    end
+
+    def release
+      released = storage.eval(RELEASE_SCRIPT, keys: ["leader"], argv: [token]).to_i == 1
+      @leader = false
+      released
+    rescue StandardError
+      @leader = false
+      false
+    end
+
+    private
+
+    attr_reader :storage, :ttl_ms
+  end
+end

data/lib/sidekiq_vigil/middleware/server.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Middleware
+    class Server
+      include Sidekiq::ServerMiddleware if defined?(Sidekiq::ServerMiddleware)
+
+      def initialize(collector: SidekiqVigil.collector, monotonic_clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+        @collector = collector
+        @monotonic_clock = monotonic_clock
+      end
+
+      def call(worker, job, queue)
+        started_at = monotonic_clock.call
+        failed = false
+        begin
+          yield
+        rescue Exception # rubocop:disable Lint/RescueException
+          failed = true
+          raise
+        ensure
+          record(worker, job, queue, started_at, failed)
+        end
+      end
+
+      private
+
+      attr_reader :collector, :monotonic_clock
+
+      def record(worker, _job, queue, started_at, failed)
+        collector.record(worker:, queue:, duration: monotonic_clock.call - started_at, failed:)
+      rescue StandardError => e
+        SidekiqVigil.logger.error("[sidekiq-vigil] collector failed: #{e.class}: #{e.message}")
+      end
+    end
+  end
+end