source_monitor 0.10.2 → 0.11.0

data/lib/generators/source_monitor/install/templates/source_monitor.rb.tt

@@ -85,10 +85,9 @@ SourceMonitor.configure do |config|
 
   # ---- Source health monitoring ---------------------------------------
   # Tune how many fetches SourceMonitor evaluates when determining health
-  # status, as well as thresholds for warnings and automatic pauses.
+  # status, as well as thresholds for automatic pauses.
   config.health.window_size = 20
-  config.health.healthy_threshold = 0.8
-  config.health.warning_threshold = 0.5
+  config.health.healthy_threshold = 0.8            # Ratio for "working" status
   config.health.auto_pause_threshold = 0.2
   config.health.auto_resume_threshold = 0.6
   config.health.auto_pause_cooldown_minutes = 60

data/lib/source_monitor/analytics/scrape_recommendations.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Analytics
+    class ScrapeRecommendations
+      def initialize(threshold: SourceMonitor.config.scraping.scrape_recommendation_threshold)
+        @threshold = threshold.to_i
+      end
+
+      def candidates_count
+        @candidates_count ||= Source.scrape_candidates(threshold: @threshold).count
+      end
+
+      def candidate_ids
+        @candidate_ids ||= Source.scrape_candidates(threshold: @threshold).pluck(:id)
+      end
+
+      def candidate?(source_id)
+        candidate_ids.include?(source_id)
+      end
+
+      private
+
+      attr_reader :threshold
+    end
+  end
+end

data/lib/source_monitor/configuration/health_settings.rb

@@ -5,7 +5,6 @@ module SourceMonitor
     class HealthSettings
       attr_accessor :window_size,
         :healthy_threshold,
-        :warning_threshold,
         :auto_pause_threshold,
         :auto_resume_threshold,
         :auto_pause_cooldown_minutes
@@ -17,7 +16,6 @@ module SourceMonitor
       def reset!
         @window_size = 20
         @healthy_threshold = 0.8
-        @warning_threshold = 0.5
         @auto_pause_threshold = 0.2
         @auto_resume_threshold = 0.6
         @auto_pause_cooldown_minutes = 60

data/lib/source_monitor/configuration/scraping_settings.rb

@@ -3,11 +3,13 @@
 module SourceMonitor
   class Configuration
     class ScrapingSettings
-      attr_accessor :max_in_flight_per_source, :max_bulk_batch_size, :min_scrape_interval
+      attr_accessor :max_in_flight_per_source, :max_bulk_batch_size, :min_scrape_interval,
+                    :scrape_recommendation_threshold
 
       DEFAULT_MAX_IN_FLIGHT = nil
       DEFAULT_MAX_BULK_BATCH_SIZE = 100
       DEFAULT_MIN_SCRAPE_INTERVAL = 1.0
+      DEFAULT_SCRAPE_RECOMMENDATION_THRESHOLD = 200
 
       def initialize
         reset!
@@ -17,6 +19,7 @@ module SourceMonitor
         @max_in_flight_per_source = DEFAULT_MAX_IN_FLIGHT
         @max_bulk_batch_size = DEFAULT_MAX_BULK_BATCH_SIZE
         @min_scrape_interval = DEFAULT_MIN_SCRAPE_INTERVAL
+        @scrape_recommendation_threshold = DEFAULT_SCRAPE_RECOMMENDATION_THRESHOLD
       end
 
       def max_in_flight_per_source=(value)
@@ -31,6 +34,10 @@ module SourceMonitor
         @min_scrape_interval = normalize_numeric_float(value)
       end
 
+      def scrape_recommendation_threshold=(value)
+        @scrape_recommendation_threshold = normalize_numeric(value)
+      end
+
       private
 
       def normalize_numeric(value)

data/lib/source_monitor/dashboard/queries/stats_query.rb

@@ -14,7 +14,9 @@ module SourceMonitor
             active_sources: integer_value(source_counts["active_sources"]),
             failed_sources: integer_value(source_counts["failed_sources"]),
             total_items: total_items_count,
-            fetches_today: fetches_today_count
+            fetches_today: fetches_today_count,
+            health_distribution: health_distribution,
+            scrape_candidates_count: scrape_candidates_count
           }
         end
 
@@ -62,6 +64,15 @@ module SourceMonitor
           reference_time.in_time_zone.beginning_of_day
         end
 
+        def health_distribution
+          raw_counts = SourceMonitor::Source.active.group(:health_status).count
+          %w[working declining improving failing].each_with_object({}) { |s, h| h[s] = raw_counts.fetch(s, 0) }
+        end
+
+        def scrape_candidates_count
+          SourceMonitor::Analytics::ScrapeRecommendations.new.candidates_count
+        end
+
         def integer_value(value)
           value.to_i
         end

data/lib/source_monitor/dashboard/queries.rb

@@ -60,10 +60,10 @@ module SourceMonitor
         end
       end
 
-      def upcoming_fetch_schedule
-        cache.fetch(:upcoming_fetch_schedule) do
+      def upcoming_fetch_schedule(pages: {})
+        cache.fetch([ :upcoming_fetch_schedule, pages ]) do
           measure(:upcoming_fetch_schedule) do
-            SourceMonitor::Dashboard::UpcomingFetchSchedule.new(scope: SourceMonitor::Source.active)
+            SourceMonitor::Dashboard::UpcomingFetchSchedule.new(scope: SourceMonitor::Source.active, pages: pages)
           end
         end
       end
@@ -110,6 +110,9 @@ module SourceMonitor
         SourceMonitor::Metrics.gauge(:dashboard_stats_failed_sources, stats[:failed_sources])
         SourceMonitor::Metrics.gauge(:dashboard_stats_total_items, stats[:total_items])
         SourceMonitor::Metrics.gauge(:dashboard_stats_fetches_today, stats[:fetches_today])
+        stats[:health_distribution]&.each do |status, count|
+          SourceMonitor::Metrics.gauge(:"dashboard_stats_health_#{status}", count)
+        end
       end
 
       def queue_name_map

data/lib/source_monitor/dashboard/recent_activity_presenter.rb

@@ -31,21 +31,22 @@ module SourceMonitor
 
       def fetch_event(event)
         domain = source_domain(event.source_feed_url)
+        label = domain ? "#{domain} \u2014 Fetch ##{event.id}" : "Fetch ##{event.id}"
         {
-          label: "Fetch ##{event.id}",
+          label: label,
           description: "#{event.items_created.to_i} created / #{event.items_updated.to_i} updated",
           status: event.success? ? :success : :failure,
           type: :fetch,
           time: event.occurred_at,
-          path: url_helpers.fetch_log_path(event.id),
-          url_display: domain,
-          url_href: event.source_feed_url
+          path: url_helpers.fetch_log_path(event.id)
         }
       end
 
       def scrape_event(event)
+        name = event.source_name.presence
+        label = name ? "#{name} \u2014 Scrape ##{event.id}" : "Scrape ##{event.id}"
         {
-          label: "Scrape ##{event.id}",
+          label: label,
           description: (event.scraper_adapter.presence || "Scraper"),
           status: event.success? ? :success : :failure,
           type: :scrape,

data/lib/source_monitor/dashboard/upcoming_fetch_schedule.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "source_monitor/pagination/paginator"
+
 module SourceMonitor
   module Dashboard
     class UpcomingFetchSchedule
@@ -12,6 +14,9 @@ module SourceMonitor
         :window_end,
         :include_unscheduled,
         :sources,
+        :page,
+        :has_next_page,
+        :has_previous_page,
         keyword_init: true
       ) do
         def empty?
@@ -27,11 +32,15 @@ module SourceMonitor
         { key: "240+", label: "240 minutes +", min_minutes: 240, max_minutes: nil, include_unscheduled: true }
       ].freeze
 
+      DEFAULT_PER_PAGE = 10
+
       attr_reader :scope, :reference_time
 
-      def initialize(scope: SourceMonitor::Source.active, reference_time: Time.current)
+      def initialize(scope: SourceMonitor::Source.active, reference_time: Time.current, pages: {}, per_page: DEFAULT_PER_PAGE)
         @scope = scope
         @reference_time = reference_time
+        @pages = pages
+        @per_page = per_page
       end
 
       def groups
@@ -40,21 +49,22 @@ module SourceMonitor
 
       private
 
+      attr_reader :pages, :per_page
+
       def build_groups
-        definitions = build_definitions
-        scheduled_sources.each do |source|
-          definition = definition_for(source.next_fetch_at)
-          definitions[definition[:key]][:sources] << source if definition
-        end
+        INTERVAL_DEFINITIONS.filter_map do |definition|
+          bucket_scope = scope_for_bucket(definition)
+          next unless bucket_scope.exists?
 
-        unscheduled_sources.each do |source|
-          definition = definitions.values.find { |value| value[:include_unscheduled] }
-          next unless definition
+          page_number = pages.fetch(definition[:key], 1).to_i
+          page_number = 1 if page_number < 1
 
-          definition[:sources] << source
-        end
+          result = SourceMonitor::Pagination::Paginator.new(
+            scope: bucket_scope.order(:next_fetch_at, :name),
+            page: page_number,
+            per_page: per_page
+          ).paginate
 
-        definitions.values.map do |definition|
           Group.new(
             key: definition[:key],
             label: definition[:label],
@@ -63,64 +73,40 @@ module SourceMonitor
             window_start: window_start_for(definition[:min_minutes]),
             window_end: window_end_for(definition[:max_minutes]),
             include_unscheduled: definition[:include_unscheduled],
-            sources: sort_sources(definition[:sources])
+            sources: result.records,
+            page: result.page,
+            has_next_page: result.has_next_page,
+            has_previous_page: result.has_previous_page
           )
         end
       end
 
-      def build_definitions
-        INTERVAL_DEFINITIONS.each_with_object({}) do |definition, memo|
-          memo[definition[:key]] = definition.merge(sources: [])
-        end
-      end
-
-      def scheduled_sources
-        scope.where.not(next_fetch_at: nil).order(:next_fetch_at)
-      end
-
-      def unscheduled_sources
-        scope.where(next_fetch_at: nil).order(:name)
-      end
-
-      def definition_for(next_fetch_at)
-        minutes = minutes_until(next_fetch_at)
-
-        INTERVAL_DEFINITIONS.find do |definition|
-          min = definition[:min_minutes]
-          max = definition[:max_minutes]
-
-          minutes >= min && (max.nil? || minutes < max)
+      def scope_for_bucket(definition)
+        window_start = reference_time + definition[:min_minutes].minutes
+        max_minutes = definition[:max_minutes]
+
+        if max_minutes.nil?
+          # Last bucket: 240+ minutes OR unscheduled (nil next_fetch_at)
+          scheduled = scope.where(next_fetch_at: window_start..)
+          unscheduled = scope.where(next_fetch_at: nil)
+          scheduled.or(unscheduled)
+        else
+          window_end = reference_time + max_minutes.minutes
+          scope.where(next_fetch_at: window_start...window_end)
         end
       end
 
-      def minutes_until(timestamp)
-        return Float::INFINITY if timestamp.blank?
-
-        minutes = (timestamp - reference_time) / 60.0
-        return 0 if minutes.negative?
-
-        minutes
-      end
-
       def window_start_for(min_minutes)
-        return nil if min_minutes.nil? || min_minutes.infinite?
+        return nil if min_minutes.nil?
 
         reference_time + min_minutes.minutes
       end
 
       def window_end_for(max_minutes)
-        return nil if max_minutes.nil? || max_minutes.infinite?
+        return nil if max_minutes.nil?
 
         reference_time + max_minutes.minutes
       end
-
-      def sort_sources(sources)
-        future_cap = reference_time + 100.years
-
-        sources.sort_by do |source|
-          [ source.next_fetch_at || future_cap, source.name.to_s ]
-        end
-      end
     end
   end
 end

data/lib/source_monitor/favicons/discoverer.rb

@@ -144,6 +144,10 @@ module SourceMonitor
         return unless body && body.bytesize > 0
         return if body.bytesize > settings.max_download_size
 
+        if content_type == "image/svg+xml"
+          return convert_svg_to_result(body, url)
+        end
+
         filename = derive_filename(url, content_type)
 
         Result.new(
@@ -156,6 +160,18 @@ module SourceMonitor
         nil
       end
 
+      def convert_svg_to_result(svg_body, url)
+        converted = SvgConverter.call(svg_body, filename: derive_filename(url, "image/svg+xml"))
+        return nil unless converted
+
+        Result.new(
+          io: converted[:io],
+          filename: converted[:filename],
+          content_type: converted[:content_type],
+          url: url
+        )
+      end
+
       def derive_filename(favicon_url, content_type)
         uri = URI.parse(favicon_url)
         basename = File.basename(uri.path) if uri.path.present?

data/lib/source_monitor/favicons/svg_converter.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Favicons
+    class SvgConverter
+      PNG_CONTENT_TYPE = "image/png"
+      DEFAULT_SIZE = 64
+
+      # Converts an SVG string to PNG bytes using MiniMagick.
+      # Returns a Hash with :io, :content_type, :filename or nil on failure.
+      def self.call(svg_body, filename: "favicon.png", size: DEFAULT_SIZE)
+        return nil unless defined?(MiniMagick)
+
+        new(svg_body, filename: filename, size: size).call
+      end
+
+      def initialize(svg_body, filename:, size:)
+        @svg_body = svg_body
+        @filename = filename.sub(/\.svg\z/i, ".png")
+        @size = size
+      end
+
+      def call
+        convert_svg_to_png
+      rescue StandardError => e
+        log_conversion_failure(e)
+        nil
+      end
+
+      private
+
+      def convert_svg_to_png
+        image = MiniMagick::Image.read(@svg_body, ".svg")
+        image.format("png")
+        image.resize("#{@size}x#{@size}")
+
+        png_bytes = image.to_blob
+
+        return nil if png_bytes.nil? || png_bytes.empty?
+
+        {
+          io: StringIO.new(png_bytes),
+          content_type: PNG_CONTENT_TYPE,
+          filename: @filename
+        }
+      ensure
+        image&.destroy!
+      end
+
+      def log_conversion_failure(error)
+        return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+        Rails.logger.warn(
+          "[SourceMonitor::Favicons::SvgConverter] SVG conversion failed: #{error.message}"
+        )
+      rescue StandardError # rubocop:disable Lint/SuppressedException
+      end
+    end
+  end
+end

data/lib/source_monitor/fetching/cloudflare_bypass.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Fetching
+    class CloudflareBypass
+      USER_AGENTS = [
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.15",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0"
+      ].freeze
+
+      CLOUDFLARE_MARKERS = FeedFetcher::CLOUDFLARE_MARKERS
+      SNIFF_LIMIT = FeedFetcher::SNIFF_LIMIT
+
+      attr_reader :response, :feed_url
+
+      def initialize(response:, feed_url:)
+        @response = response
+        @feed_url = feed_url
+      end
+
+      def call
+        attempt_cookie_replay || attempt_ua_rotation
+      end
+
+      private
+
+      def attempt_cookie_replay
+        cookies = extract_cookies(response)
+        return if cookies.blank?
+
+        headers = { "Cookie" => cookies, "Cache-Control" => "no-cache", "Pragma" => "no-cache" }
+        result = fetch_with_headers(headers)
+        result unless cloudflare_blocked?(result)
+      end
+
+      def attempt_ua_rotation
+        USER_AGENTS.each do |ua|
+          headers = {
+            "User-Agent" => ua,
+            "Cache-Control" => "no-cache",
+            "Pragma" => "no-cache"
+          }
+          result = fetch_with_headers(headers)
+          return result unless cloudflare_blocked?(result)
+        end
+
+        nil
+      end
+
+      def fetch_with_headers(headers)
+        client = SourceMonitor::HTTP.client(headers: headers, retry_requests: false)
+        client.get(feed_url)
+      rescue StandardError
+        nil
+      end
+
+      def cloudflare_blocked?(response)
+        return true if response.nil?
+
+        body = response.body
+        return true if body.blank?
+
+        snippet = body[0, SNIFF_LIMIT].downcase
+        CLOUDFLARE_MARKERS.any? { |marker| snippet.include?(marker.downcase) }
+      end
+
+      def extract_cookies(resp)
+        set_cookie = resp&.headers&.dig("set-cookie")
+        return if set_cookie.blank?
+
+        Array(set_cookie).filter_map { |cookie|
+          cookie.to_s.split(";").first.presence
+        }.join("; ").presence
+      end
+    end
+  end
+end

data/lib/source_monitor/fetching/feed_fetcher/source_updater.rb

@@ -4,6 +4,17 @@ module SourceMonitor
   module Fetching
     class FeedFetcher
       class SourceUpdater
+        CONSECUTIVE_FAILURE_PAUSE_THRESHOLD = 5
+
+        ERROR_CATEGORY_MAP = {
+          SourceMonitor::Fetching::TimeoutError => "network",
+          SourceMonitor::Fetching::ConnectionError => "network",
+          SourceMonitor::Fetching::ParsingError => "parse",
+          SourceMonitor::Fetching::BlockedError => "blocked",
+          SourceMonitor::Fetching::AuthenticationError => "auth",
+          SourceMonitor::Fetching::UnexpectedResponseError => "unknown"
+        }.freeze
+
         attr_reader :source, :adaptive_interval
 
         def initialize(source:, adaptive_interval:)
@@ -19,6 +30,7 @@ module SourceMonitor
             last_error: nil,
             last_error_at: nil,
             failure_count: 0,
+            consecutive_fetch_failures: 0,
             feed_format: derive_feed_format(feed)
           }
 
@@ -47,7 +59,8 @@ module SourceMonitor
             last_http_status: response.status,
             last_error: nil,
             last_error_at: nil,
-            failure_count: 0
+            failure_count: 0,
+            consecutive_fetch_failures: 0
           }
 
           if (etag = response.headers["etag"] || response.headers["ETag"])
@@ -74,13 +87,15 @@ module SourceMonitor
             last_http_status: error.http_status,
             last_error: error.message,
             last_error_at: now,
-            failure_count: source.failure_count.to_i + 1
+            failure_count: source.failure_count.to_i + 1,
+            consecutive_fetch_failures: source.consecutive_fetch_failures.to_i + 1
           }
 
           adaptive_interval.apply_adaptive_interval!(attrs, content_changed: false, failure: true)
           attrs[:metadata] = updated_metadata
           decision = apply_retry_strategy!(attrs, error, now)
           source.update!(attrs)
+          check_consecutive_failure_auto_pause!
           decision
         end
 
@@ -101,6 +116,7 @@ module SourceMonitor
             error_class: error&.class&.name,
             error_message: error&.message,
             error_backtrace: error_backtrace(error),
+            error_category: categorize_error(error),
             metadata: feed_metadata(feed, error: error, feed_signature: feed_signature, item_errors: item_errors)
           )
         end
@@ -139,6 +155,56 @@ module SourceMonitor
           attributes[:fetch_circuit_until] = nil
         end
 
+        def check_consecutive_failure_auto_pause!
+          return if source.consecutive_fetch_failures < CONSECUTIVE_FAILURE_PAUSE_THRESHOLD
+          return if source.auto_paused_until.present? && source.auto_paused_until.future?
+
+          now = Time.current
+          cooldown = [ SourceMonitor.config.health.auto_pause_cooldown_minutes.to_i, 1 ].max
+          pause_until = now + cooldown.minutes
+
+          source.update_columns(
+            auto_paused_until: pause_until,
+            auto_paused_at: now,
+            health_status: "failing",
+            health_status_changed_at: now,
+            backoff_until: pause_until,
+            next_fetch_at: pause_until
+          )
+
+          notify_auto_pause(now)
+        rescue StandardError => error
+          Rails.logger.error(
+            "[SourceMonitor::SourceUpdater] Auto-pause check failed for source #{source.id}: #{error.message}"
+          ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        end
+
+        def notify_auto_pause(timestamp)
+          message = "Source '#{source.name}' auto-paused after #{CONSECUTIVE_FAILURE_PAUSE_THRESHOLD} consecutive fetch failures"
+
+          source.fetch_logs.create!(
+            success: false,
+            started_at: timestamp,
+            completed_at: timestamp,
+            duration_ms: 0,
+            http_status: nil,
+            error_class: "SourceMonitor::AutoPause",
+            error_message: message,
+            metadata: { event: "auto_pause", consecutive_failures: source.consecutive_fetch_failures }
+          )
+
+          SourceMonitor::Realtime.broadcast_toast(
+            message: "#{message}.",
+            level: :warning,
+            delay_ms: 8000
+          )
+          SourceMonitor::Realtime.broadcast_source(source)
+        rescue StandardError => error
+          Rails.logger.warn(
+            "[SourceMonitor::SourceUpdater] Auto-pause notification failed for source #{source.id}: #{error.message}"
+          ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        end
+
         def enqueue_favicon_fetch_if_needed
           return unless defined?(ActiveStorage)
           return unless SourceMonitor.config.favicons.enabled?
@@ -192,6 +258,20 @@ module SourceMonitor
           nil
         end
 
+        def categorize_error(error)
+          return if error.nil?
+
+          if error.is_a?(SourceMonitor::Fetching::HTTPError)
+            status = error.status.to_i
+            return "auth" if status == 401 || status == 403
+            return "network"
+          end
+
+          ERROR_CATEGORY_MAP.fetch(error.class) do
+            error.is_a?(SourceMonitor::Fetching::FetchError) ? "unknown" : nil
+          end
+        end
+
         def derive_feed_format(feed)
           return unless feed