sidekiq-vigil 0.1.0

data/lib/sidekiq_vigil/alert/event.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Alert
+    class Event
+      attr_reader :result, :transition, :state, :history, :timestamp
+
+      def initialize(result:, transition:, state:, history:, timestamp:)
+        @result = result
+        @transition = transition.to_sym
+        @state = state
+        @history = history
+        @timestamp = timestamp
+      end
+
+      def severity
+        result.severity
+      end
+
+      def alert_id
+        result.alert_id
+      end
+
+      def to_h
+        {
+          schema_version: "1.0",
+          event: transition,
+          alert_id:,
+          timestamp: timestamp.utc.iso8601,
+          alert: result.to_h,
+          state: state.to_h,
+          history:
+        }
+      end
+    end
+
+    class DigestEvent
+      attr_reader :events, :timestamp
+
+      def initialize(events:, timestamp:, limit: 5)
+        @events = events
+        @timestamp = timestamp
+        @limit = limit
+      end
+
+      def transition
+        :digest
+      end
+
+      def severity
+        severities.include?(:critical) || severities.include?(:error) ? :critical : :warn
+      end
+
+      def alert_id
+        "digest"
+      end
+
+      def result
+        @result ||= Result.new(
+          check_name: "digest",
+          severity:,
+          value: events.length,
+          message: "#{events.length} alerts in this cycle",
+          metadata: { counts: counts, top: top }
+        )
+      end
+
+      def history
+        []
+      end
+
+      def state
+        State.new(status: "firing", severity: severity.to_s)
+      end
+
+      def to_h
+        {
+          schema_version: "1.0",
+          event: "digest",
+          alert_id:,
+          timestamp: timestamp.utc.iso8601,
+          counts:,
+          alerts: top
+        }
+      end
+
+      private
+
+      attr_reader :limit
+
+      def severities
+        events.map(&:severity)
+      end
+
+      def counts
+        events.group_by(&:severity).transform_values(&:length)
+      end
+
+      def top
+        events.sort_by { |event| severity_rank(event.severity) }.first(limit).map do |event|
+          event.result.to_h
+        end
+      end
+
+      def severity_rank(severity)
+        { error: 0, critical: 1, warn: 2, ok: 3 }.fetch(severity, 4)
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/alert/grouping.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Alert
+    module Grouping
+      module_function
+
+      def apply(events, threshold:, timestamp:, limit: 5)
+        return events if events.length <= threshold
+
+        [DigestEvent.new(events:, timestamp:, limit:)]
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/alert/manager.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module SidekiqVigil
+  module Alert
+    class Manager
+      HISTORY_TTL = 24 * 60 * 60
+
+      def initialize(storage:, config:, mute:, clock: -> { Time.now })
+        @storage = storage
+        @config = config
+        @mute = mute
+        @clock = clock
+      end
+
+      def process(results)
+        persisted = storage.hash_get_all("alerts")
+        active_ids = results.map(&:alert_id)
+        muted = mute.active?
+        events = results.filter_map { |result| process_result(result, persisted, muted) }
+        prune(persisted.keys - active_ids)
+        Grouping.apply(events, threshold: config.group_threshold, timestamp: now, limit: config.group_top_n)
+      end
+
+      private
+
+      attr_reader :storage, :config, :mute, :clock
+
+      def process_result(result, persisted, muted)
+        state = State.load(persisted.fetch(result.alert_id, "{}"))
+        refresh_flapping(state)
+        record_history(result)
+        event = transition(result, state)
+        event ||= unsuppressed_event(result, state) unless muted
+        state.suppressed = true if event && muted
+        persist(result.alert_id, state)
+        muted ? nil : event
+      end
+
+      def transition(result, state)
+        case state.status
+        when "ok" then transition_from_ok(result, state)
+        when "pending" then transition_from_pending(result, state)
+        when "firing" then transition_from_firing(result, state)
+        when "resolved" then transition_from_resolved(result, state)
+        else reset_invalid_state(result, state)
+        end
+      end
+
+      def transition_from_ok(result, state)
+        return update_ok(state, result) if result.ok?
+
+        begin_pending(state, result)
+        fire(result, state, :firing) if state.cycles >= config.pending_cycles
+      end
+
+      def transition_from_pending(result, state)
+        return update_ok(state, result) if result.ok?
+
+        state.cycles += 1
+        update_observation(state, result)
+        return if flapping_active?(state)
+
+        fire(result, state, :firing) if state.cycles >= config.pending_cycles
+      end
+
+      def transition_from_firing(result, state)
+        if result.ok?
+          previous_severity = state.severity
+          change_status(state, "resolved")
+          state.suppressed = false
+          state.severity = previous_severity
+          return build_event(result, :resolved, state) if config.resolve_notice
+
+          return nil
+        end
+
+        state.cycles += 1
+        previous_severity = state.severity
+        effective = escalated_result(result, state)
+        update_observation(state, effective)
+        return fire(effective, state, :escalated) if effective.severity.to_s != previous_severity
+        return unless cooldown_elapsed?(state)
+
+        fire(effective, state, :still_firing)
+      end
+
+      def transition_from_resolved(result, state)
+        return update_ok(state, result) if result.ok?
+
+        begin_pending(state, result)
+        if flapping?(state)
+          state.flapping_until = now.to_f + config.flap_window
+          return if state.flap_notified
+
+          state.flap_notified = true
+          return build_event(result, :flapping, state)
+        end
+
+        fire(result, state, :firing) if state.cycles >= config.pending_cycles
+      end
+
+      def begin_pending(state, result)
+        change_status(state, "pending")
+        state.cycles = 1
+        state.first_seen_at = now.to_f
+        update_observation(state, result)
+      end
+
+      def fire(result, state, transition)
+        change_status(state, "firing")
+        state.last_notified_at = now.to_f
+        state.suppressed = false
+        state.flapping_until = nil
+        build_event(result, transition, state)
+      end
+
+      def update_ok(state, result)
+        change_status(state, "ok")
+        state.cycles = 0
+        state.first_seen_at = nil
+        state.severity = "ok"
+        state.suppressed = false
+        update_observation(state, result)
+        nil
+      end
+
+      def update_observation(state, result)
+        state.severity = result.severity.to_s
+        state.value = result.value
+        state.threshold = result.threshold
+        state.message = result.message
+      end
+
+      def escalated_result(result, state)
+        threshold = config.escalate_after
+        return result unless threshold && result.severity == :warn && state.cycles >= threshold
+
+        Result.new(
+          **result.to_h,
+          severity: :critical,
+          metadata: result.metadata.merge(escalated: true)
+        )
+      end
+
+      def cooldown_elapsed?(state)
+        now.to_f - state.last_notified_at.to_f >= config.cooldown
+      end
+
+      def unsuppressed_event(result, state)
+        return unless state.firing? && state.suppressed
+
+        state.suppressed = false
+        fire(result, state, :unmuted)
+      end
+
+      def build_event(result, transition, state)
+        Event.new(
+          result:,
+          transition:,
+          state:,
+          history: storage.list_range("history:#{result.alert_id}").map { |item| JSON.parse(item) },
+          timestamp: now
+        )
+      end
+
+      def record_history(result)
+        point = JSON.generate(timestamp: now.to_f, value: result.value, severity: result.severity)
+        storage.list_push("history:#{result.alert_id}", point, ttl: HISTORY_TTL)
+      end
+
+      def persist(alert_id, state)
+        storage.managed_hash_write("alerts", alert_id, state.dump)
+      end
+
+      def prune(alert_ids)
+        storage.hash_delete("alerts", *alert_ids)
+        alert_ids.each { |alert_id| storage.delete("history:#{alert_id}") }
+      end
+
+      def reset_invalid_state(result, state)
+        state.status = "ok"
+        transition_from_ok(result, state)
+      end
+
+      def change_status(state, status)
+        return if state.status == status
+
+        state.status = status
+        state.last_transition_at = now.to_f
+        state.transition_timestamps << now.to_f
+        prune_transition_timestamps(state)
+      end
+
+      def flapping?(state)
+        return false unless config.flap_window.positive? && config.flap_threshold.positive?
+
+        prune_transition_timestamps(state)
+        state.transition_timestamps.length >= config.flap_threshold
+      end
+
+      def flapping_active?(state)
+        state.flapping_until && now.to_f < state.flapping_until
+      end
+
+      def refresh_flapping(state)
+        prune_transition_timestamps(state)
+        return if flapping_active?(state)
+        return if state.transition_timestamps.length >= config.flap_threshold
+
+        state.flapping_until = nil
+        state.flap_notified = false
+      end
+
+      def prune_transition_timestamps(state)
+        cutoff = now.to_f - config.flap_window
+        state.transition_timestamps.select! { |timestamp| timestamp.to_f >= cutoff }
+      end
+
+      def now
+        clock.call
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/alert/mute.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SidekiqVigil
+  module Alert
+    class Mute
+      def initialize(storage:, schedules: [], clock: -> { Time.now }, timezone: "UTC")
+        @storage = storage
+        @schedules = schedules
+        @clock = clock
+        @timezone = timezone
+      end
+
+      def active?
+        !reason.nil?
+      end
+
+      def reason
+        dynamic_reason || scheduled_reason
+      end
+
+      def mute(duration, reason: nil)
+        payload = JSON.generate(reason: reason || "manual mute", until: clock.call.to_f + duration)
+        storage.set("mute", payload, ttl: duration)
+      end
+
+      def unmute
+        storage.delete("mute")
+      end
+
+      private
+
+      attr_reader :storage, :schedules, :clock, :timezone
+
+      def dynamic_reason
+        payload = storage.get("mute")
+        return unless payload
+
+        JSON.parse(payload).fetch("reason")
+      rescue JSON::ParserError
+        "manual mute"
+      end
+
+      def scheduled_reason
+        schedules.each do |schedule|
+          duration = schedule.fetch(:duration)
+          matched = active_schedule?(schedule.fetch(:cron), duration)
+          return schedule.fetch(:reason, "scheduled maintenance") if matched
+        end
+        nil
+      end
+
+      def active_schedule?(expression, duration)
+        current = clock.call
+        current_minute = Time.at((current.to_f / 60).floor * 60)
+        cron = Cron.new(expression)
+        minute_count = (duration / 60.0).ceil
+
+        (0..minute_count).any? do |offset|
+          start_time = current_minute - (offset * 60)
+          next false unless current.to_f < start_time.to_f + duration
+
+          cron.match?(Timezone.local_time(start_time, timezone))
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/alert/state.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SidekiqVigil
+  module Alert
+    class State
+      ATTRIBUTES = %i[
+        status cycles first_seen_at last_notified_at last_transition_at severity value threshold message suppressed
+        flap_notified flapping_until transition_timestamps
+      ].freeze
+
+      attr_accessor(*ATTRIBUTES)
+
+      def initialize(attributes = {})
+        attributes = attributes.transform_keys(&:to_sym)
+        @status = attributes.fetch(:status, "ok")
+        @cycles = attributes.fetch(:cycles, 0)
+        @first_seen_at = attributes[:first_seen_at]
+        @last_notified_at = attributes[:last_notified_at]
+        @last_transition_at = attributes[:last_transition_at]
+        @severity = attributes.fetch(:severity, "ok")
+        @value = attributes[:value]
+        @threshold = attributes[:threshold]
+        @message = attributes[:message]
+        @suppressed = attributes.fetch(:suppressed, false)
+        @flap_notified = attributes.fetch(:flap_notified, false)
+        @flapping_until = attributes[:flapping_until]
+        @transition_timestamps = attributes.fetch(:transition_timestamps, [])
+      end
+
+      def self.load(json)
+        new(JSON.parse(json))
+      rescue JSON::ParserError
+        new
+      end
+
+      def to_h
+        ATTRIBUTES.to_h { |name| [name, public_send(name)] }
+      end
+
+      def dump
+        JSON.generate(to_h)
+      end
+
+      def firing?
+        status == "firing"
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/base.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class Base
+      attr_reader :storage, :options, :clock, :logger, :api
+
+      def initialize(storage:, options: {}, clock: -> { Time.now }, logger: SidekiqVigil.logger, api: SidekiqApi.new)
+        @storage = storage
+        @options = options
+        @clock = clock
+        @logger = logger
+        @api = api
+      end
+
+      def execute
+        Array(call)
+      rescue StandardError => e
+        logger.error("[sidekiq-vigil] #{check_name} failed: #{e.class}: #{e.message}")
+        [Result.new(
+          check_name:,
+          severity: :error,
+          message: "#{e.class}: #{e.message}",
+          metadata: { exception_class: e.class.name }
+        )]
+      end
+
+      def call
+        raise NotImplementedError, "#{self.class} must implement #call"
+      end
+
+      def check_name
+        class_name = self.class.name || "custom_check"
+        class_name.split("::").last.gsub(/([a-z])([A-Z])/, '\1_\2').downcase
+      end
+
+      private
+
+      def threshold_result(target:, value:, warn:, critical:, message: nil, metadata: {})
+        severity, threshold = severity_and_threshold(value, warn, critical)
+        Result.new(check_name:, target:, severity:, value:, threshold:, message:, metadata:)
+      end
+
+      def severity_and_threshold(value, warn, critical)
+        return [:critical, critical] if critical && value >= critical
+        return [:warn, warn] if warn && value >= warn
+
+        [:ok, warn]
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/failure_rate.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class FailureRate < Base
+      def call
+        processed, failed = counts
+        rate = processed.zero? ? 0.0 : failed.fdiv(processed)
+        return insufficient_samples(processed, rate) if processed < options.fetch(:min_samples, 20)
+
+        threshold_result(
+          target: "global",
+          value: rate,
+          warn: options.fetch(:warn, 0.05),
+          critical: options.fetch(:critical, 0.20),
+          message: format("%<rate>.1f%% failures (%<failed>d/%<processed>d attempts)", rate: rate * 100, failed:, processed:),
+          metadata: { processed:, failed: }
+        )
+      end
+
+      private
+
+      def counts
+        minute_count = (options.fetch(:window, 300) / 60.0).ceil
+        minute_count.times.each_with_object([0, 0]) do |offset, totals|
+          key = "stats:#{(clock.call - (offset * 60)).utc.strftime('%Y%m%d%H%M')}"
+          stats = storage.hash_get_all(key)
+          totals[0] += stats.fetch("processed", 0).to_i
+          totals[1] += stats.fetch("failed", 0).to_i
+        end
+      end
+
+      def insufficient_samples(processed, rate)
+        Result.new(
+          check_name:,
+          severity: :ok,
+          value: rate,
+          threshold: options.fetch(:warn, 0.05),
+          message: "insufficient samples (#{processed}/#{options.fetch(:min_samples, 20)})",
+          metadata: { processed:, insufficient_samples: true }
+        )
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/memory.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class Memory < Base
+      def call
+        return [skipped_result] if RUBY_PLATFORM.match?(/mswin|mingw/)
+
+        keys = storage.scan("mem:*")
+        return [empty_result] if keys.empty?
+
+        keys.map do |key|
+          process_id = key.delete_prefix(storage.prefix).delete_prefix("mem:")
+          value_mb = storage.get("mem:#{process_id}").to_f / 1024
+          threshold_result(
+            target: process_id,
+            value: value_mb,
+            warn: options.fetch(:warn_mb, 1_500),
+            critical: options.fetch(:critical_mb, 2_500),
+            message: format("%.1f MiB RSS", value_mb)
+          )
+        end
+      end
+
+      private
+
+      def skipped_result
+        Result.new(
+          check_name:,
+          severity: :ok,
+          message: "memory collection is unsupported on Windows",
+          metadata: { skipped: true }
+        )
+      end
+
+      def empty_result
+        Result.new(check_name:, severity: :ok, value: 0, message: "no process RSS reports")
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/process_alive.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class ProcessAlive < Base
+      def call
+        processes = api.processes
+        minimum = options.fetch(:min_processes, 1)
+        active = processes.count { |process| active?(process) }
+        severity = active < minimum ? :critical : :ok
+
+        Result.new(
+          check_name:,
+          severity:,
+          value: active,
+          threshold: minimum,
+          message: "#{active}/#{minimum} required processes alive",
+          metadata: { total: processes.size }
+        )
+      end
+
+      private
+
+      def active?(process)
+        quiet_since = process_value(process, "quiet_since")
+        return clock.call.to_f - quiet_since.to_f < quiet_threshold if quiet_since
+
+        unless stopping?(process)
+          clear_quiet_state(process)
+          return true
+        end
+
+        state_key = "check_state:quiet:#{process_identity(process)}"
+        observed_at = storage.get(state_key)
+        storage.set(state_key, observed_at || clock.call.to_f, ttl: quiet_threshold + 3_600)
+        observed_at.nil? || clock.call.to_f - observed_at.to_f < quiet_threshold
+      end
+
+      def process_value(process, key)
+        process[key] || process[key.to_sym]
+      end
+
+      def stopping?(process)
+        return process.stopping? if process.respond_to?(:stopping?)
+
+        %w[true 1].include?(process_value(process, "quiet").to_s)
+      end
+
+      def clear_quiet_state(process)
+        storage.delete("check_state:quiet:#{process_identity(process)}")
+      end
+
+      def process_identity(process)
+        return process.identity if process.respond_to?(:identity)
+
+        process_value(process, "identity") || process.object_id
+      end
+
+      def quiet_threshold
+        options.fetch(:quiet_threshold, 900)
+      end
+    end
+  end
+end

data/lib/sidekiq_vigil/check/queue_latency.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SidekiqVigil
+  module Check
+    class QueueLatency < Base
+      def call
+        api.queues(options.fetch(:queues, :all)).map do |queue|
+          thresholds = options.merge(options.fetch(:per_queue, {}).fetch(queue.name, {}))
+          latency = queue.latency.to_f
+          threshold_result(
+            target: queue.name,
+            value: latency,
+            warn: thresholds.fetch(:warn, 60),
+            critical: thresholds.fetch(:critical, 300),
+            message: "oldest job has waited #{latency.round(2)}s"
+          )
+        end
+      end
+    end
+  end
+end