sidekiq-vigil 0.1.0

data/lib/sidekiq_vigil/notifier/base.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Notifier
+    class Base
+      attr_reader :options, :logger
+
+      def initialize(options: {}, logger: SidekiqVigil.logger, **_dependencies)
+        @options = options
+        @logger = logger
+      end
+
+      def notify(_event)
+        raise NotImplementedError, "#{self.class} must implement #notify"
+      end
+
+      def inspect
+        "#<#{self.class} options=[FILTERED]>"
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/notifier/http_transport.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module SidekiqVigil
+  module Notifier
+    class HttpTransport
+      Response = Data.define(:code, :body) do
+        def success?
+          code.to_i.between?(200, 299)
+        end
+      end
+
+      def post(url, body, headers: {})
+        uri = URI.parse(url)
+        raise Error, "notification URL must use HTTP or HTTPS" unless %w[http https].include?(uri.scheme)
+
+        request = Net::HTTP::Post.new(uri.request_uri, { "Content-Type" => "application/json" }.merge(headers))
+        request.body = body
+        response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+          http.open_timeout = 5
+          http.read_timeout = 10
+          http.request(request)
+        end
+        Response.new(code: response.code, body: response.body)
+      rescue URI::InvalidURIError, SocketError, SystemCallError, Timeout::Error => e
+        raise Error, "notification request failed (#{e.class})"
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/notifier/log.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SidekiqVigil
+  module Notifier
+    class Log < Base
+      def notify(event)
+        logger.warn("[sidekiq-vigil] #{JSON.generate(event.to_h)}")
+        true
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/notifier/manager.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Notifier
+    class Manager
+      def initialize(notifiers:, logger: SidekiqVigil.logger)
+        @notifiers = notifiers
+        @logger = logger
+      end
+
+      def notify(events)
+        Array(events).each do |event|
+          notifiers.each { |notifier| notify_one(notifier, event) }
+        end
+      end
+
+      private
+
+      attr_reader :notifiers, :logger
+
+      def notify_one(notifier, event)
+        notifier.notify(event)
+      rescue StandardError => e
+        logger.error("[sidekiq-vigil] #{notifier.class} failed: #{e.class}: #{e.message}")
+      end
+    end
+
+    module Factory
+      CLASSES = {
+        log: Log,
+        slack: Slack,
+        webhook: Webhook
+      }.freeze
+
+      module_function
+
+      def build(config, logger: SidekiqVigil.logger)
+        config.runnable_notifiers.map do |definition|
+          klass = definition.klass || CLASSES.fetch(definition.name)
+          klass.new(options: definition.options, logger:)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/notifier/slack.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+require "json"
+require "socket"
+
+module SidekiqVigil
+  module Notifier
+    class Slack < Base
+      SPARKS = "▁▂▃▄▅▆▇█"
+
+      def initialize(
+        options: {},
+        logger: SidekiqVigil.logger,
+        transport: HttpTransport.new,
+        sleeper: ->(seconds) { sleep(seconds) },
+        fallback: Log.new(logger:)
+      )
+        super(options:, logger:)
+        @transport = transport
+        @sleeper = sleeper
+        @fallback = fallback
+      end
+
+      def notify(event)
+        attempts = options.fetch(:retries, 3) + 1
+        attempts.times do |attempt|
+          response = transport.post(webhook_url(event), JSON.generate(payload(event)))
+          return true if response.success?
+
+          raise Error, "Slack returned HTTP #{response.code}"
+        rescue StandardError => e
+          if attempt == attempts - 1
+            logger.error("[sidekiq-vigil] Slack notification failed after #{attempts} attempts: #{safe_error(e)}")
+            fallback.notify(event)
+            return false
+          end
+          sleeper.call(2**attempt)
+        end
+      end
+
+      def payload(event)
+        {
+          text: header(event),
+          blocks: event.transition == :digest ? digest_blocks(event) : alert_blocks(event)
+        }
+      end
+
+      private
+
+      attr_reader :transport, :sleeper, :fallback
+
+      def webhook_url(event)
+        routes = options.fetch(:routes, {})
+        routes[event.severity] || routes[event.severity.to_s] || options.fetch(:webhook_url)
+      end
+
+      def header(event)
+        return "📦 DIGEST — #{event.result.message}" if event.transition == :digest
+        return "✅ RESOLVED — #{event.result.check_name}" if event.transition == :resolved
+
+        icon = %i[critical error].include?(event.severity) ? "🔥" : "⚠️"
+        "#{icon} #{event.severity.to_s.upcase} — #{event.result.check_name}"
+      end
+
+      def fields(event)
+        result = event.result
+        [
+          markdown_field("*Check*\n#{display(result.check_name)}"),
+          markdown_field("*Target*\n#{result.target}"),
+          markdown_field("*Value / threshold*\n#{display(result.value)} / #{display(result.threshold)}"),
+          markdown_field("*Duration*\n#{duration(event)}"),
+          markdown_field("*Environment*\n#{environment}"),
+          markdown_field("*Host*\n#{host}")
+        ]
+      end
+
+      def context_block(event)
+        parts = ["Event #{event.transition}"]
+        spark = sparkline(event.history)
+        parts << "Trend #{spark}" unless spark.empty?
+        parts << "<#{options[:web_ui_url]}|Open Sidekiq>" if options[:web_ui_url]
+        mentions = options.fetch(:mention, {})
+        mention = mentions[event.severity] || mentions[event.severity.to_s]
+        parts << mention if mention
+
+        { type: "context", elements: [{ type: "mrkdwn", text: parts.join(" • ") }] }
+      end
+
+      def alert_blocks(event)
+        [
+          header_block(event),
+          { type: "section", fields: fields(event) },
+          message_block(event.result.message),
+          context_block(event)
+        ].compact
+      end
+
+      def digest_blocks(event)
+        metadata = event.result.metadata
+        [
+          header_block(event),
+          { type: "section", text: { type: "mrkdwn", text: digest_counts(metadata.fetch(:counts)) } },
+          { type: "section", text: { type: "mrkdwn", text: digest_top(metadata.fetch(:top)) } },
+          context_block(event)
+        ]
+      end
+
+      def header_block(event)
+        { type: "header", text: { type: "plain_text", text: header(event) } }
+      end
+
+      def message_block(message)
+        return unless message
+
+        { type: "section", text: { type: "mrkdwn", text: "*Message*\n#{message}" } }
+      end
+
+      def digest_counts(counts)
+        labels = { error: "Error", critical: "Critical", warn: "Warn", ok: "OK" }
+        values = labels.filter_map do |severity, label|
+          count = counts[severity] || counts[severity.to_s]
+          "#{label}: #{count}" if count.to_i.positive?
+        end
+        "*Severity counts*\n#{values.join(' • ')}"
+      end
+
+      def digest_top(alerts)
+        lines = alerts.map do |alert|
+          severity = fetch_value(alert, :severity).to_s.upcase
+          check = fetch_value(alert, :check_name)
+          target = fetch_value(alert, :target)
+          value = display(fetch_value(alert, :value))
+          threshold = display(fetch_value(alert, :threshold))
+          "• *#{severity}* #{check} / #{target} — #{value} / #{threshold}"
+        end
+        "*Top alerts*\n#{lines.join("\n")}"
+      end
+
+      def markdown_field(text)
+        { type: "mrkdwn", text: }
+      end
+
+      def sparkline(history)
+        values = history.filter_map { |point| numeric(point["value"] || point[:value]) }
+        return "" if values.empty?
+        return SPARKS[0] * values.length if values.min == values.max
+
+        values.map do |value|
+          index = ((value - values.min) / (values.max - values.min) * (SPARKS.length - 1)).round
+          SPARKS[index]
+        end.join
+      end
+
+      def numeric(value)
+        Float(value)
+      rescue TypeError, ArgumentError
+        nil
+      end
+
+      def display(value)
+        value.nil? ? "-" : value.to_s
+      end
+
+      def duration(event)
+        started_at = event.state.first_seen_at
+        return "-" unless started_at
+
+        seconds = [event.timestamp.to_f - started_at.to_f, 0].max.round
+        return "#{seconds}s" if seconds < 60
+        return "#{seconds / 60}m #{seconds % 60}s" if seconds < 3_600
+
+        "#{seconds / 3_600}h #{(seconds % 3_600) / 60}m"
+      end
+
+      def environment
+        options.fetch(:environment) { ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "unknown" }
+      end
+
+      def host
+        options.fetch(:host) { Socket.gethostname }
+      end
+
+      def fetch_value(hash, key)
+        hash[key] || hash[key.to_s]
+      end
+
+      def safe_error(error)
+        error.message.gsub(%r{https?://\S+}, "[FILTERED]")
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/notifier/webhook.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SidekiqVigil
+  module Notifier
+    class Webhook < Base
+      def initialize(options: {}, logger: SidekiqVigil.logger, transport: HttpTransport.new)
+        super(options:, logger:)
+        @transport = transport
+      end
+
+      def notify(event)
+        response = transport.post(options.fetch(:url), JSON.generate(event.to_h), headers: options.fetch(:headers, {}))
+        raise Error, "webhook returned HTTP #{response.code}" unless response.success?
+
+        true
+      end
+
+      private
+
+      attr_reader :transport
+    end
+  end
+end

data/lib/sidekiq_vigil/reporter.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "digest"
+require "socket"
+
+module SidekiqVigil
+  class Reporter
+    STATS_TTL = 8 * 24 * 60 * 60
+    EXECUTION_TTL = 60 * 60
+    CONFIG_TTL = 60 * 60
+
+    attr_reader :thread
+
+    def initialize(storage:, collector:, config:, logger: SidekiqVigil.logger, sleeper: ->(seconds) { sleep(seconds) })
+      @storage = storage
+      @collector = collector
+      @config = config
+      @logger = logger
+      @sleeper = sleeper
+      @running = false
+    end
+
+    def start
+      return thread if thread&.alive?
+
+      @running = true
+      @thread = Thread.new { run }
+      @thread.name = "sidekiq-vigil-reporter" if @thread.respond_to?(:name=)
+      @thread
+    end
+
+    def stop
+      @running = false
+      thread&.wakeup if thread&.status == "sleep"
+      thread&.join(config.flush_interval + 1)
+      @thread = nil
+    end
+
+    def flush_once
+      snapshot = collector.drain
+      flush_snapshot(snapshot)
+      report_rss
+      storage.hash_write("config_digest", { process_id => config.digest }, ttl: CONFIG_TTL)
+      true
+    rescue StandardError => e
+      collector.merge(snapshot) if snapshot
+      logger.error("[sidekiq-vigil] reporter flush failed: #{e.class}: #{e.message}")
+      false
+    end
+
+    def rss_kb
+      return linux_rss if File.readable?("/proc/self/status")
+      return macos_rss if RUBY_PLATFORM.include?("darwin")
+
+      nil
+    end
+
+    private
+
+    attr_reader :storage, :collector, :config, :logger, :sleeper
+
+    def run
+      while @running
+        flush_once
+        sleeper.call(config.flush_interval)
+      end
+    rescue StandardError => e
+      logger.error("[sidekiq-vigil] reporter thread failed: #{e.class}: #{e.message}")
+    end
+
+    def flush_snapshot(snapshot)
+      snapshot.stats.each do |minute, fields|
+        storage.hash_increment("stats:#{minute}", fields, ttl: STATS_TTL)
+      end
+      snapshot.executions.each do |queue, values|
+        storage.hash_increment("exec:#{queue}", values.transform_keys(&:to_s), ttl: EXECUTION_TTL)
+      end
+    end
+
+    def report_rss
+      rss = rss_kb
+      return unless rss
+
+      storage.set("mem:#{process_id}", rss, ttl: config.flush_interval * 3)
+    end
+
+    def process_id
+      @process_id ||= "#{Socket.gethostname}:#{Process.pid}"
+    end
+
+    def linux_rss
+      match = File.read("/proc/self/status").match(/^VmRSS:\s+(\d+)\s+kB/)
+      match && Integer(match[1])
+    end
+
+    def macos_rss
+      output = IO.popen(["ps", "-o", "rss=", "-p", Process.pid.to_s], &:read)
+      Integer(output.strip, exception: false)
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/sidekiq_vigil/result.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  class Result
+    SEVERITIES = %i[ok warn critical error].freeze
+
+    attr_reader :check_name, :target, :severity, :value, :threshold, :message, :metadata
+
+    def initialize(check_name:, severity:, target: "global", value: nil, threshold: nil, message: nil, metadata: {})
+      severity = severity.to_sym
+      raise ArgumentError, "unknown severity: #{severity}" unless SEVERITIES.include?(severity)
+
+      @check_name = check_name.to_s
+      @target = target.to_s
+      @severity = severity
+      @value = value
+      @threshold = threshold
+      @message = message
+      @metadata = metadata.freeze
+      freeze
+    end
+
+    def ok?
+      severity == :ok
+    end
+
+    def alert_id
+      "#{check_name}:#{target}"
+    end
+
+    def to_h
+      {
+        check_name: check_name,
+        target: target,
+        severity: severity,
+        value: value,
+        threshold: threshold,
+        message: message,
+        metadata: metadata
+      }
+    end
+  end
+end

data/lib/sidekiq_vigil/runtime.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  class Runtime
+    attr_reader :reporter, :checker
+
+    def initialize(
+      config:,
+      storage:,
+      collector: SidekiqVigil.collector,
+      logger: SidekiqVigil.logger,
+      reporter: nil,
+      checker: nil
+    )
+      @config = config
+      @reporter = reporter || Reporter.new(storage:, collector:, config:, logger:)
+      @checker = checker || Checker.new(storage:, config:, logger:)
+    end
+
+    def start
+      return false unless config.enabled?
+
+      reporter.start
+      checker.start
+      true
+    rescue StandardError => e
+      SidekiqVigil.logger.error("[sidekiq-vigil] startup failed: #{e.class}: #{e.message}")
+      false
+    end
+
+    def quiet
+      checker.stop
+    end
+
+    def stop
+      checker.stop
+      reporter.stop
+    end
+
+    private
+
+    attr_reader :config
+  end
+end

data/lib/sidekiq_vigil/sidekiq_api.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  class SidekiqApi
+    def queues(names = :all)
+      return Sidekiq::Queue.all if names == :all
+
+      Array(names).map { |name| Sidekiq::Queue.new(name.to_s) }
+    end
+
+    def retry_size
+      Sidekiq::RetrySet.new.size
+    end
+
+    def dead_size
+      Sidekiq::DeadSet.new.size
+    end
+
+    def processes
+      Sidekiq::ProcessSet.new.to_a
+    end
+
+    def workers
+      Sidekiq::Workers.new.map do |process_id, thread_id, work|
+        {
+          process_id:,
+          thread_id:,
+          work: {
+            "run_at" => work.run_at.to_f,
+            "payload" => work.payload
+          }
+        }
+      end
+    end
+
+    def scheduled
+      Sidekiq::ScheduledSet.new.to_a
+    end
+  end
+end