anomonitor 0.6.1

data/lib/anomonitor/configuration.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module Anomonitor
+  class Configuration
+    attr_accessor :webhook_url, :poll_interval, :cooldown, :retention_days,
+                  :dashboard_path, :dashboard_base_url,
+                  :tenants, :exclude_tenants, :tenant_switch,
+                  :schema_drift_exclude, :schema_drift_interval,
+                  :authenticate, :notifier, :poll_lock,
+                  :digest_interval, :digest_last_flushed_at, :notifier_rate_limit
+
+    attr_reader :collectors, :tables, :alerts, :poll_mode, :auto_start
+
+    def initialize
+      @webhook_url = nil
+      @notifier = nil
+      @poll_interval = 60
+      @cooldown = 15 * 60
+      @retention_days = 7
+      @dashboard_path = "/anomonitor"
+      @dashboard_base_url = nil
+      @schema_drift_interval = 15 * 60
+      @authenticate = nil
+      @poll_lock = true
+      @digest_interval = nil
+      @digest_last_flushed_at = nil
+      @notifier_rate_limit = 0
+      @poll_mode = :thread
+      @auto_start = true
+      @collectors = CollectorsConfig.new
+      @tables = []
+      @alerts = []
+
+      @tenants = nil
+      @exclude_tenants = %w[public]
+      @tenant_switch = nil
+
+      @schema_drift_exclude = %w[
+        schema_migrations
+        ar_internal_metadata
+        anomonitor_*
+      ]
+    end
+
+    def anomaly_dashboard_url(anomaly_id)
+      path = "#{dashboard_path.to_s.sub(%r{/+\z}, "")}/anomalies/#{anomaly_id}"
+      path = "/#{path}" unless path.start_with?("/")
+      base = dashboard_base_url.to_s.strip.sub(%r{/+\z}, "")
+      base.empty? ? path : "#{base}#{path}"
+    end
+
+    def build_notifier
+      Notifiers.build(self)
+    end
+
+    def digest_enabled?
+      digest_interval.to_i.positive?
+    end
+
+    # Persist a mute (DB) — metric/rule/source/tenant are optional matchers.
+    # duration: seconds or ActiveSupport duration (e.g. 24.hours)
+    def mute(duration:, metric: nil, rule: nil, source: nil, tenant: nil, reason: nil)
+      seconds = duration.respond_to?(:to_i) ? duration.to_i : duration
+      Mute.create!(
+        metric: metric&.to_s,
+        rule: rule&.to_s,
+        source: source&.to_s,
+        tenant: tenant&.to_s,
+        muted_until: Time.current + seconds,
+        reason: reason
+      )
+    end
+
+    def poll_mode=(mode)
+      mode = mode.to_sym
+      unless %i[thread cron].include?(mode)
+        raise ArgumentError, "poll_mode must be :thread or :cron (got #{mode.inspect})"
+      end
+
+      @poll_mode = mode
+      @auto_start = (mode == :thread)
+    end
+
+    def auto_start=(value)
+      @auto_start = !!value
+      @poll_mode = @auto_start ? :thread : :cron
+    end
+
+    def table(name, &block)
+      source = TableSource.new(name)
+      yield source if block_given?
+      @tables << source
+      source
+    end
+
+    def alert(metric, **options)
+      rule = AlertRule.new(metric, **options)
+      @alerts << rule
+      rule
+    end
+
+    class CollectorsConfig
+      attr_accessor :sidekiq, :delayed_job, :solid_queue, :schema_drift
+
+      def initialize
+        @sidekiq = true
+        @delayed_job = true
+        @solid_queue = true
+        @schema_drift = false
+      end
+    end
+
+    class TableSource
+      attr_accessor :name, :model, :timestamp, :status, :active, :tenant, :style
+
+      def initialize(name)
+        @name = name
+        @timestamp = :created_at
+        @status = :status
+        @active = %w[pending running]
+        @tenant = nil
+        @style = :status
+      end
+
+      def model_class
+        model.to_s.constantize
+      end
+
+      def delayed_job_style?
+        style.to_s == "delayed_job"
+      end
+    end
+
+    class AlertRule
+      attr_reader :metric, :max, :window, :multiplier, :severity, :match
+
+      def initialize(metric, max: nil, window: nil, multiplier: nil, severity: "high", match: {})
+        @metric = metric.to_sym
+        @max = max
+        @window = normalize_duration(window)
+        @multiplier = multiplier
+        @severity = severity
+        @match = (match || {}).transform_keys(&:to_sym)
+      end
+
+      def threshold?
+        !max.nil?
+      end
+
+      def spike?
+        !multiplier.nil?
+      end
+
+      def window_seconds
+        @window || (5 * 60)
+      end
+
+      # match: { queue: true } => tag present; { queue: nil } => tag absent; { queue: "x" } => exact
+      def matches_point?(point)
+        return true if match.nil? || match.empty?
+
+        match.all? do |key, expected|
+          tag = point.tags[key] || point.tags[key.to_s]
+          case expected
+          when true then !tag.nil? && tag.to_s != ""
+          when false, nil then tag.nil? || tag.to_s == ""
+          else
+            tag.to_s == expected.to_s
+          end
+        end
+      end
+
+      private
+
+      def normalize_duration(value)
+        return nil if value.nil?
+        return value if value.is_a?(Numeric)
+        return value.to_i if value.respond_to?(:to_i) && !value.is_a?(String)
+
+        value
+      end
+    end
+  end
+end

data/lib/anomonitor/configuration_validator.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Anomonitor
+  class ConfigurationValidator
+    def self.warnings(config = Anomonitor.config)
+      new(config).warnings
+    end
+
+    def initialize(config)
+      @config = config
+    end
+
+    def warnings
+      list = []
+      if @config.notifier.nil? && @config.webhook_url.to_s.strip.empty?
+        list << "No notifier configured (set c.webhook_url or c.notifier) — anomalies will record as webhook failed"
+      end
+      if @config.collectors.schema_drift
+        tenants = begin
+          Tenancy.tenant_names
+        rescue StandardError
+          []
+        end
+        if tenants.size < 2
+          list << "schema_drift is enabled but fewer than 2 tenants are configured"
+        end
+      end
+      if @config.poll_mode == :thread
+        list << "poll_mode=:thread can double-collect under multi-worker Puma/Unicorn — prefer :cron or enable poll_lock"
+      end
+      path = @config.dashboard_path.to_s
+      list << "dashboard_path should start with / (got #{path.inspect})" if path != "" && !path.start_with?("/")
+      list
+    end
+
+    def log!
+      warnings.each { |w| Anomonitor.logger.warn("[Anomonitor] Config: #{w}") }
+    end
+  end
+end

data/lib/anomonitor/detector.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module Anomonitor
+  class Detector
+    def initialize(notifier: nil)
+      @notifier = notifier || Anomonitor.config.build_notifier
+    end
+
+    def evaluate(points)
+      @active_sticky_keys = []
+      @observed_sticky_bases = []
+      anomalies = []
+
+      unless points.empty?
+        Anomonitor.config.alerts.each do |rule|
+          points.each do |point|
+            next unless metric_matches?(rule, point)
+            next unless rule.matches_point?(point)
+
+            anomaly = detect(rule, point)
+            anomalies << anomaly if anomaly
+          end
+        end
+      end
+
+      resolve_sticky_anomalies(@active_sticky_keys, @observed_sticky_bases)
+      Digester.flush_if_due!(notifier: @notifier) if Anomonitor.config.digest_enabled?
+      anomalies
+    end
+
+    private
+
+    def metric_matches?(rule, point)
+      point.metric.to_s == rule.metric.to_s ||
+        (rule.spike? && %w[growth_rate queue_depth].include?(point.metric.to_s) && rule.metric.to_s == "growth_spike") ||
+        (rule.threshold? && point.metric.to_s == rule.metric.to_s)
+    end
+
+    def detect(rule, point)
+      if rule.threshold? && point.metric.to_s == rule.metric.to_s
+        note_sticky_observation(point) if point.sticky?
+        return nil unless point.value > rule.max
+
+        create_anomaly(rule, point, threshold: rule.max, reason: "threshold")
+      elsif rule.spike?
+        detect_spike(rule, point)
+      end
+    end
+
+    def detect_spike(rule, point)
+      previous = previous_value(point)
+      return nil if previous.nil? || previous <= 0
+
+      ratio = point.value / previous.to_f
+      return nil unless ratio >= rule.multiplier
+
+      create_anomaly(
+        rule,
+        point,
+        threshold: previous * rule.multiplier,
+        reason: "growth_spike",
+        extra: { previous: previous, ratio: ratio.round(2) }
+      )
+    end
+
+    def previous_value(point)
+      tags_previous = point.tags[:previous] || point.tags["previous"]
+      return tags_previous.to_f if tags_previous
+
+      window = spike_window_seconds
+      queue = point.tags[:queue] || point.tags["queue"]
+      tenant = point.tags[:tenant] || point.tags["tenant"]
+
+      scope = Anomonitor::MetricSample
+              .where(source: point.source, metric: point.metric)
+              .where("sampled_at >= ? AND sampled_at < ?", (window * 2).seconds.ago, window.seconds.ago)
+              .order(sampled_at: :desc)
+
+      # Pull a wider window then filter tags in Ruby (portable across JSON/text adapters)
+      candidates = scope.limit(100).to_a
+      if queue
+        candidates.select! { |s| (s.tags || {})["queue"] == queue || (s.tags || {})[:queue] == queue }
+      end
+      if tenant
+        candidates.select! { |s| (s.tags || {})["tenant"] == tenant || (s.tags || {})[:tenant] == tenant }
+      end
+
+      values = candidates.first(5).map(&:value)
+      return nil if values.empty?
+
+      values.sum / values.size.to_f
+    rescue StandardError
+      nil
+    end
+
+    def spike_window_seconds
+      rule = Anomonitor.config.alerts.find(&:spike?)
+      rule ? rule.window_seconds.to_i : 300
+    end
+
+    def create_anomaly(rule, point, threshold:, reason:, extra: {})
+      key = "#{reason}:#{point.cooldown_key}"
+      @active_sticky_keys << key if point.sticky?
+
+      return nil if cooling_down?(key, point)
+
+      tenant = point.tags[:tenant] || point.tags["tenant"]
+      if Mute.muted?(metric: point.metric, rule: reason, source: point.source, tenant: tenant)
+        Anomonitor.logger.info("[Anomonitor] Muted anomaly #{reason}:#{point.source}/#{point.metric}")
+        return nil
+      end
+
+      anomaly = Anomonitor::Anomaly.create!(
+        rule: reason,
+        source: point.source,
+        metric: point.metric,
+        value: point.value,
+        threshold: threshold,
+        severity: rule.severity,
+        cooldown_key: key,
+        tags: point.tags.merge(extra),
+        sampled_at: point.sampled_at,
+        webhook_status: "pending",
+        resolved_at: nil
+      )
+
+      if Anomonitor.config.digest_enabled?
+        Digester.record(anomaly)
+      else
+        delivered = @notifier.deliver(anomaly)
+        anomaly.update!(
+          webhook_status: delivered ? "delivered" : "failed",
+          webhook_delivered_at: delivered ? Time.current : nil
+        )
+      end
+      anomaly
+    rescue StandardError => e
+      Anomonitor.logger.warn("[Anomonitor] Failed to record anomaly: #{e.message}")
+      nil
+    end
+
+    def cooling_down?(key, point = nil)
+      if point&.sticky?
+        return true if Anomonitor::Anomaly.where(cooldown_key: key, resolved_at: nil).exists?
+
+        # Manual ack: stay silent for this fingerprint until a clear poll marks cleared_at
+        latest = Anomonitor::Anomaly.where(cooldown_key: key).order(created_at: :desc).first
+        latest&.manual_resolve? && !latest.cleared_after_ack?
+      else
+        cooldown = Anomonitor.config.cooldown.to_i
+        Anomonitor::Anomaly
+          .where(cooldown_key: key)
+          .where("created_at >= ?", cooldown.seconds.ago)
+          .exists?
+      end
+    end
+
+    def note_sticky_observation(point)
+      @observed_sticky_bases << sticky_base(point)
+    end
+
+    def sticky_base(point)
+      tenant = point.tags[:tenant] || point.tags["tenant"]
+      [point.source, point.metric, tenant].compact.join(":")
+    end
+
+    def anomaly_sticky_base(anomaly)
+      tags = anomaly.tags.is_a?(Hash) ? anomaly.tags : {}
+      tenant = tags["tenant"] || tags[:tenant]
+      [anomaly.source, anomaly.metric, tenant].compact.join(":")
+    end
+
+    # Resolve open schema-drift alerts only when that tenant/metric was observed this tick
+    # and the fingerprint is no longer active (cleared or items changed).
+    def resolve_sticky_anomalies(active_keys, observed_bases)
+      return if observed_bases.empty?
+
+      bases = observed_bases.uniq
+      Anomonitor::Anomaly.where(source: "schema_drift").find_each do |anomaly|
+        next unless bases.include?(anomaly_sticky_base(anomaly))
+        next if active_keys.include?(anomaly.cooldown_key)
+
+        if anomaly.open?
+          anomaly.update!(resolved_at: Time.current)
+          @notifier.deliver(anomaly, event: Notifiers::RESOLVED)
+        elsif anomaly.manual_resolve? && !anomaly.cleared_after_ack?
+          anomaly.merge_tags!("cleared_at" => Time.current.iso8601)
+          anomaly.save!
+        end
+      end
+    rescue StandardError => e
+      Anomonitor.logger.warn("[Anomonitor] Failed to resolve sticky anomalies: #{e.message}")
+    end
+  end
+end

data/lib/anomonitor/digester.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Anomonitor
+  # Batches anomaly.detected notifications into a single anomaly.digest payload.
+  class Digester
+    def self.record(anomaly)
+      new.record(anomaly)
+    end
+
+    def self.flush_if_due!(notifier: nil)
+      new(notifier: notifier).flush_if_due!
+    end
+
+    def initialize(notifier: nil)
+      @notifier = notifier || Anomonitor.config.build_notifier
+    end
+
+    def record(anomaly)
+      anomaly.update!(webhook_status: "queued")
+      true
+    rescue StandardError => e
+      Anomonitor.logger.warn("[Anomonitor] Digester record failed: #{e.message}")
+      false
+    end
+
+    def flush_if_due!
+      interval = Anomonitor.config.digest_interval.to_i
+      return 0 if interval <= 0
+
+      last = Anomonitor.config.digest_last_flushed_at
+      return 0 if last && Time.current - last < interval
+
+      flush!
+    end
+
+    def flush!
+      queued = Anomonitor::Anomaly.where(webhook_status: "queued").order(:created_at).limit(100).to_a
+      return 0 if queued.empty?
+
+      path = Anomonitor.config.dashboard_path.to_s.sub(%r{/+\z}, "")
+      base = Anomonitor.config.dashboard_base_url.to_s.strip.sub(%r{/+\z}, "")
+      dash = base.empty? ? "#{path}/anomalies" : "#{base}#{path}/anomalies"
+
+      payload = {
+        gem: "anomonitor",
+        event: Notifiers::DIGEST,
+        count: queued.size,
+        dashboard_url: dash,
+        anomalies: queued.map { |a| Notifiers.payload(a, event: Notifiers::DETECTED) }
+      }
+
+      delivered = deliver_digest(payload)
+      queued.each do |anomaly|
+        anomaly.update!(
+          webhook_status: delivered ? "delivered" : "failed",
+          webhook_delivered_at: delivered ? Time.current : nil
+        )
+      end
+      Anomonitor.config.digest_last_flushed_at = Time.current
+      queued.size
+    rescue StandardError => e
+      Anomonitor.logger.warn("[Anomonitor] Digester flush failed: #{e.message}")
+      0
+    end
+
+    private
+
+    def deliver_digest(payload)
+      if @notifier.respond_to?(:deliver_digest)
+        @notifier.deliver_digest(payload)
+      else
+        false
+      end
+    rescue StandardError => e
+      Anomonitor.logger.warn("[Anomonitor] Digest deliver error: #{e.message}")
+      false
+    end
+  end
+end

data/lib/anomonitor/engine.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Anomonitor
+  class Engine < ::Rails::Engine
+    isolate_namespace Anomonitor
+
+    config.generators do |g|
+      g.test_framework :minitest
+    end
+
+    initializer "anomonitor.assets" do |app|
+      if app.config.respond_to?(:assets)
+        app.config.assets.precompile += %w[anomonitor/application.css]
+      end
+    end
+
+    # Migrations are installed into the host via:
+    #   rails anomonitor:install:migrations
+    # Do not also append the engine migrate path (duplicates + breaks
+    # Apartment apps that wrap migrations with public-only guards).
+
+    config.after_initialize do
+      Anomonitor::ConfigurationValidator.new(Anomonitor.config).log!
+
+      # :thread starts an in-process poller; :cron relies on `rails anomonitor:poll`
+      next unless Anomonitor.config.poll_mode == :thread
+      next unless Anomonitor.config.auto_start
+      next if defined?(Rails::Console)
+      next if File.basename($PROGRAM_NAME).include?("rake")
+      next if Rails.env.test?
+
+      Anomonitor::Poller.instance.start
+    end
+  end
+end

data/lib/anomonitor/jobs/browser.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Anomonitor
+  module Jobs
+    # Fetches live job rows from every enabled job collector backend.
+    class Browser
+      DEFAULT_LIMIT = 100
+      STATUSES = %w[all pending failed locked].freeze
+
+      def self.fetch(filters = {})
+        new(filters).fetch
+      end
+
+      def self.enabled_sources
+        new.enabled_sources
+      end
+
+      def self.status_options(source = nil)
+        if source.to_s.start_with?("table:")
+          name = source.to_s.delete_prefix("table:")
+          table = Anomonitor.config.tables.find { |t| t.name.to_s == name }
+          return %w[all pending failed] if table && !table.delayed_job_style?
+        end
+
+        STATUSES
+      end
+
+      def initialize(filters = {})
+        @filters = normalize(filters)
+      end
+
+      def fetch
+        rows = []
+        adapters.each do |adapter|
+          next if @filters[:source] && adapter.source_key != @filters[:source]
+
+          rows.concat(Array(adapter.fetch(@filters)))
+        rescue StandardError => e
+          Anomonitor.logger.warn("[Anomonitor] Jobs browser (#{adapter.source_key}) error: #{e.message}")
+        end
+
+        rows = filter_query(rows)
+        rows.sort_by { |r| r.sort_at }.reverse.first(@filters[:limit])
+      end
+
+      def enabled_sources
+        adapters.map(&:source_key)
+      end
+
+      private
+
+      def filter_query(rows)
+        q = @filters[:q]
+        return rows if q.nil? || q.empty?
+
+        needle = q.downcase
+        rows.select { |r| r.name.to_s.downcase.include?(needle) || r.id.to_s.downcase.include?(needle) }
+      end
+
+      def normalize(filters)
+        raw = filters.respond_to?(:to_unsafe_h) ? filters.to_unsafe_h : filters
+        raw = raw.to_h.transform_keys(&:to_sym)
+        status = raw[:status].to_s
+        status = "all" unless STATUSES.include?(status)
+
+        {
+          source: raw[:source].presence,
+          status: status,
+          tenant: raw[:tenant].presence,
+          queue: raw[:queue].presence,
+          q: raw[:q].to_s.strip.presence,
+          limit: (raw[:limit] || DEFAULT_LIMIT).to_i.clamp(1, 500)
+        }
+      end
+
+      def adapters
+        list = []
+        cfg = Anomonitor.config.collectors
+        list << Browsers::Sidekiq.new if cfg.sidekiq
+        list << Browsers::DelayedJob.new if cfg.delayed_job
+        list << Browsers::SolidQueue.new if cfg.solid_queue
+        Anomonitor.config.tables.each do |table|
+          list << Browsers::Table.new(table)
+        end
+        list
+      end
+    end
+  end
+end