source_monitor 0.10.2 → 0.11.0

data/lib/source_monitor/fetching/feed_fetcher.rb

@@ -78,7 +78,7 @@ module SourceMonitor
       def perform_fetch(started_at, instrumentation_payload)
         response = perform_request
         handle_response(response, started_at, instrumentation_payload)
-      rescue TimeoutError, ConnectionError, HTTPError, ParsingError => error
+      rescue TimeoutError, ConnectionError, HTTPError, ParsingError, BlockedError, AuthenticationError => error
         raise error
       rescue Faraday::TimeoutError => error
         raise TimeoutError.new(error.message, original_error: error)
@@ -209,12 +209,66 @@ module SourceMonitor
         )
       end
 
+      SNIFF_LIMIT = 4096
+
+      CLOUDFLARE_MARKERS = [
+        "<title>Just a moment</title>",
+        "<title>Attention Required</title>",
+        "cf-challenge",
+        "cf-browser-verification",
+        "__cf_chl_",
+        "data-ray="
+      ].freeze
+
+      CAPTCHA_MARKERS = [
+        "g-recaptcha",
+        "h-captcha"
+      ].freeze
+
+      LOGIN_TITLE_PATTERN = /<title>\s*(log\s*in|sign\s*in)\s*<\/title>/i
+
       def parse_feed(body, response)
+        blocked_by = detect_blocked_response(body, response)
+
+        if blocked_by == "cloudflare" && !@bypass_attempted
+          @bypass_attempted = true
+          bypass_response = CloudflareBypass.new(response: response, feed_url: source.feed_url).call
+          if bypass_response
+            body = bypass_response.body
+          else
+            raise BlockedError.new(blocked_by: blocked_by, response: response)
+          end
+        elsif blocked_by
+          raise BlockedError.new(blocked_by: blocked_by, response: response)
+        end
+
         Feedjira.parse(body)
+      rescue BlockedError
+        raise
       rescue StandardError => error
         raise ParsingError.new(error.message, response: response, original_error: error)
       end
 
+      def detect_blocked_response(body, _response)
+        return if body.blank?
+
+        snippet = body[0, SNIFF_LIMIT]
+        snippet_lower = snippet.downcase
+
+        return "cloudflare" if CLOUDFLARE_MARKERS.any? { |marker| snippet_lower.include?(marker.downcase) }
+        return "captcha" if CAPTCHA_MARKERS.any? { |marker| snippet_lower.include?(marker.downcase) }
+
+        if snippet_lower.match?(LOGIN_TITLE_PATTERN)
+          return "login_wall"
+        end
+
+        if snippet_lower.include?("<html") && snippet_lower.include?("<form") && snippet_lower.include?("password")
+          return "login_wall"
+        end
+
+        nil
+      end
+
       def handle_failure(error, started_at:, instrumentation_payload:)
         response = error.response
         body = response&.body

data/lib/source_monitor/fetching/fetch_error.rb

@@ -84,5 +84,32 @@ module SourceMonitor
         "Unexpected response received"
       end
     end
+
+    class BlockedError < FetchError
+      CODE = "blocked"
+
+      attr_reader :blocked_by
+
+      def initialize(message = nil, blocked_by: "unknown", **kwargs)
+        @blocked_by = blocked_by
+        super(message, **kwargs)
+      end
+
+      protected
+
+      def default_message
+        "Feed blocked by #{blocked_by}"
+      end
+    end
+
+    class AuthenticationError < FetchError
+      CODE = "authentication"
+
+      protected
+
+      def default_message
+        "Authentication required"
+      end
+    end
   end
 end

data/lib/source_monitor/fetching/fetch_runner.rb

@@ -41,6 +41,10 @@ module SourceMonitor
         source = resolve_source(source_or_id)
         return unless source
 
+        if force && source.fetch_status == "fetching"
+          return :already_fetching
+        end
+
         # Don't broadcast here - controller handles immediate UI update
         source.update_columns(fetch_status: "queued")
         SourceMonitor::FetchFeedJob.perform_later(source.id, force: force)

data/lib/source_monitor/fetching/retry_policy.rb

@@ -21,6 +21,8 @@ module SourceMonitor
         http_5xx: { attempts: 2, wait: 10.minutes, circuit_wait: 90.minutes },
         http_4xx: { attempts: 1, wait: 45.minutes, circuit_wait: 2.hours },
         parsing: { attempts: 1, wait: 30.minutes, circuit_wait: 2.hours },
+        blocked: { attempts: 1, wait: 1.hour, circuit_wait: 4.hours },
+        authentication: { attempts: 1, wait: 1.hour, circuit_wait: 4.hours },
         unexpected: { attempts: 1, wait: 30.minutes, circuit_wait: 2.hours },
         fallback: { attempts: 2, wait: 10.minutes, circuit_wait: 90.minutes }
       }.freeze
@@ -76,6 +78,8 @@ module SourceMonitor
         end
 
         return :parsing if error.is_a?(SourceMonitor::Fetching::ParsingError)
+        return :blocked if error.is_a?(SourceMonitor::Fetching::BlockedError)
+        return :authentication if error.is_a?(SourceMonitor::Fetching::AuthenticationError)
         return :unexpected if error.is_a?(SourceMonitor::Fetching::UnexpectedResponseError)
 
         :fallback

data/lib/source_monitor/health/import_source_health_check.rb

@@ -11,19 +11,19 @@ module SourceMonitor
       end
 
       def call
-        return Result.new(status: "unhealthy", error_message: "Missing feed URL", http_status: nil) if feed_url.blank?
+        return Result.new(status: "failing", error_message: "Missing feed URL", http_status: nil) if feed_url.blank?
 
         response = connection.get(feed_url)
         status_code = response_status(response)
         healthy = healthy_status?(status_code)
 
         Result.new(
-          status: healthy ? "healthy" : "unhealthy",
+          status: healthy ? "working" : "failing",
           error_message: healthy ? nil : error_for_status(status_code),
           http_status: status_code
         )
       rescue StandardError => error
-        Result.new(status: "unhealthy", error_message: error.message, http_status: response_status(error))
+        Result.new(status: "failing", error_message: error.message, http_status: response_status(error))
       end
 
       private

data/lib/source_monitor/health/source_health_monitor.rb

@@ -32,6 +32,7 @@ module SourceMonitor
           auto_paused_at = nil
           attrs[:auto_paused_until] = nil
           attrs[:auto_paused_at] = nil
+          attrs[:consecutive_fetch_failures] = 0
           attrs[:backoff_until] = nil if source.backoff_until.present?
         end
 
@@ -114,19 +115,17 @@ module SourceMonitor
         end
       end
 
-      def determine_status(rate, auto_paused_until, logs)
-        if auto_paused_active?(auto_paused_until)
-          "auto_paused"
+      def determine_status(rate, _auto_paused_until, logs)
+        if rate >= healthy_threshold
+          "working"
+        elsif rate < auto_pause_threshold
+          "failing"
         elsif consecutive_failures(logs) >= 3
           "declining"
         elsif improving_streak?(logs)
           "improving"
-        elsif rate >= healthy_threshold
-          "healthy"
-        elsif rate >= warning_threshold
-          "warning"
         else
-          "critical"
+          "declining"
         end
       end
 
@@ -143,7 +142,7 @@ module SourceMonitor
       end
 
       def apply_status(attrs, status)
-        previous = source.health_status.presence || "healthy"
+        previous = source.health_status.presence || "working"
         return if previous == status
 
         attrs[:health_status] = status
@@ -161,11 +160,7 @@ module SourceMonitor
       end
 
       def healthy_threshold
-        [ config.healthy_threshold.to_f, warning_threshold ].max
-      end
-
-      def warning_threshold
-        config.warning_threshold.to_f
+        config.healthy_threshold.to_f
       end
 
       def auto_paused_active?(value)

data/lib/source_monitor/health/source_health_reset.rb

@@ -26,7 +26,7 @@ module SourceMonitor
 
       def reset_attributes
         {
-          health_status: "healthy",
+          health_status: "working",
           auto_paused_at: nil,
           auto_paused_until: nil,
           rolling_success_rate: nil,

data/lib/source_monitor/pagination/paginator.rb

@@ -8,6 +8,7 @@ module SourceMonitor
       :per_page,
       :has_next_page,
       :has_previous_page,
+      :total_count,
       keyword_init: true
     ) do
       def has_next_page?
@@ -29,6 +30,12 @@ module SourceMonitor
 
         [ page - 1, 1 ].max
       end
+
+      def total_pages
+        return 1 if total_count.nil? || total_count <= 0
+
+        [ 1, (total_count.to_f / per_page).ceil ].max
+      end
     end
 
     class Paginator
@@ -41,6 +48,7 @@ module SourceMonitor
       end
 
       def paginate
+        total = compute_total_count
         paginated_records = fetch_records
         has_next_page = paginated_records.length > per_page
 
@@ -49,7 +57,8 @@ module SourceMonitor
           page: page,
           per_page: per_page,
           has_next_page: has_next_page,
-          has_previous_page: page > 1
+          has_previous_page: page > 1,
+          total_count: total
         )
       end
 
@@ -57,6 +66,14 @@ module SourceMonitor
 
       attr_reader :scope, :page, :per_page
 
+      def compute_total_count
+        if scope.is_a?(ActiveRecord::Relation)
+          scope.count
+        else
+          Array(scope).size
+        end
+      end
+
       def fetch_records
         offset = (page - 1) * per_page
 

data/lib/source_monitor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SourceMonitor
-  VERSION = "0.10.2"
+  VERSION = "0.11.0"
 end

data/lib/source_monitor.rb

@@ -54,6 +54,7 @@ module SourceMonitor
   autoload :Assets, "source_monitor/assets"
 
   module Analytics
+    autoload :ScrapeRecommendations, "source_monitor/analytics/scrape_recommendations"
     autoload :SourceFetchIntervalDistribution, "source_monitor/analytics/source_fetch_interval_distribution"
     autoload :SourceActivityRates, "source_monitor/analytics/source_activity_rates"
     autoload :SourcesIndexMetrics, "source_monitor/analytics/sources_index_metrics"
@@ -81,6 +82,7 @@ module SourceMonitor
     autoload :RetryPolicy, "source_monitor/fetching/retry_policy"
     autoload :StalledFetchReconciler, "source_monitor/fetching/stalled_fetch_reconciler"
     autoload :AdvisoryLock, "source_monitor/fetching/advisory_lock"
+    autoload :CloudflareBypass, "source_monitor/fetching/cloudflare_bypass"
   end
 
   module ImportSessions
@@ -90,6 +92,7 @@ module SourceMonitor
 
   module Favicons
     autoload :Discoverer, "source_monitor/favicons/discoverer"
+    autoload :SvgConverter, "source_monitor/favicons/svg_converter"
   end
 
   module Images

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: source_monitor
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.11.0
 platform: ruby
 authors:
 - dchuk
@@ -238,6 +238,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".claude/agent-memory/vbw-vbw-debugger/MEMORY.md"
 - ".claude/agent-memory/vbw-vbw-dev/MEMORY.md"
 - ".claude/agent-memory/vbw-vbw-lead/MEMORY.md"
 - ".claude/agents/rails-concern.md"
@@ -372,9 +373,11 @@ files:
 - app/controllers/concerns/.keep
 - app/controllers/concerns/source_monitor/sanitizes_search_params.rb
 - app/controllers/source_monitor/application_controller.rb
+- app/controllers/source_monitor/bulk_scrape_enablements_controller.rb
 - app/controllers/source_monitor/dashboard_controller.rb
 - app/controllers/source_monitor/fetch_logs_controller.rb
 - app/controllers/source_monitor/health_controller.rb
+- app/controllers/source_monitor/import_history_dismissals_controller.rb
 - app/controllers/source_monitor/import_sessions/bulk_configuration.rb
 - app/controllers/source_monitor/import_sessions/entry_annotation.rb
 - app/controllers/source_monitor/import_sessions/health_check_management.rb
@@ -389,6 +392,7 @@ files:
 - app/controllers/source_monitor/source_health_checks_controller.rb
 - app/controllers/source_monitor/source_health_resets_controller.rb
 - app/controllers/source_monitor/source_retries_controller.rb
+- app/controllers/source_monitor/source_scrape_tests_controller.rb
 - app/controllers/source_monitor/source_turbo_responses.rb
 - app/controllers/source_monitor/sources_controller.rb
 - app/helpers/source_monitor/application_helper.rb
@@ -422,6 +426,7 @@ files:
 - app/views/source_monitor/dashboard/_fetch_schedule.html.erb
 - app/views/source_monitor/dashboard/_job_metrics.html.erb
 - app/views/source_monitor/dashboard/_recent_activity.html.erb
+- app/views/source_monitor/dashboard/_scrape_recommendations.html.erb
 - app/views/source_monitor/dashboard/_stat_card.html.erb
 - app/views/source_monitor/dashboard/_stats.html.erb
 - app/views/source_monitor/dashboard/index.html.erb
@@ -444,7 +449,11 @@ files:
 - app/views/source_monitor/items/show.html.erb
 - app/views/source_monitor/logs/index.html.erb
 - app/views/source_monitor/scrape_logs/show.html.erb
+- app/views/source_monitor/shared/_pagination.html.erb
 - app/views/source_monitor/shared/_toast.html.erb
+- app/views/source_monitor/source_scrape_tests/_result.html.erb
+- app/views/source_monitor/source_scrape_tests/show.html.erb
+- app/views/source_monitor/sources/_bulk_scrape_enable_modal.html.erb
 - app/views/source_monitor/sources/_bulk_scrape_form.html.erb
 - app/views/source_monitor/sources/_bulk_scrape_modal.html.erb
 - app/views/source_monitor/sources/_details.html.erb
@@ -492,6 +501,10 @@ files:
 - db/migrate/20260210204022_add_composite_index_to_log_entries.rb
 - db/migrate/20260222120000_add_min_scrape_interval_to_sources.rb
 - db/migrate/20260222194201_add_word_counts_to_item_contents.rb
+- db/migrate/20260305120000_add_dismissed_at_to_import_histories.rb
+- db/migrate/20260306233004_add_error_category_to_fetch_logs.rb
+- db/migrate/20260307120000_add_consecutive_fetch_failures_to_sources.rb
+- db/migrate/20260312120000_simplify_health_status_values.rb
 - docs/configuration.md
 - docs/deployment.md
 - docs/gh-cli-workflow.md
@@ -503,6 +516,7 @@ files:
 - lib/generators/source_monitor/install/install_generator.rb
 - lib/generators/source_monitor/install/templates/source_monitor.rb.tt
 - lib/source_monitor.rb
+- lib/source_monitor/analytics/scrape_recommendations.rb
 - lib/source_monitor/analytics/source_activity_rates.rb
 - lib/source_monitor/analytics/source_fetch_interval_distribution.rb
 - lib/source_monitor/analytics/sources_index_metrics.rb
@@ -536,8 +550,10 @@ files:
 - lib/source_monitor/engine.rb
 - lib/source_monitor/events.rb
 - lib/source_monitor/favicons/discoverer.rb
+- lib/source_monitor/favicons/svg_converter.rb
 - lib/source_monitor/feedjira_extensions.rb
 - lib/source_monitor/fetching/advisory_lock.rb
+- lib/source_monitor/fetching/cloudflare_bypass.rb
 - lib/source_monitor/fetching/completion/event_publisher.rb
 - lib/source_monitor/fetching/completion/follow_up_handler.rb
 - lib/source_monitor/fetching/completion/retention_handler.rb