source_monitor 0.11.1 → 0.12.0

data/app/views/source_monitor/sources/new.html.erb

@@ -1,16 +1,7 @@
 <div class="mx-auto max-w-2xl py-10">
   <h1 class="text-3xl font-semibold">New Source</h1>
 
-  <% if @source.errors.any? %>
-    <div class="mt-4 rounded border border-red-300 bg-red-50 p-4">
-      <h2 class="font-medium text-red-700">Please fix the following:</h2>
-      <ul class="mt-2 list-disc space-y-1 pl-5 text-red-700">
-        <% @source.errors.full_messages.each do |message| %>
-          <li><%= message %></li>
-        <% end %>
-      </ul>
-    </div>
-  <% end %>
+  <%= render "source_monitor/shared/form_errors", record: @source %>
 
   <div class="mt-6">
     <%= render "form", source: @source %>

data/config/routes.rb

@@ -17,7 +17,7 @@ SourceMonitor::Engine.routes.draw do
     end
   end
   resources :items, only: %i[index show] do
-    post :scrape, on: :member
+    resource :scrape, only: :create, controller: "item_scrapes"
   end
   resources :bulk_scrape_enablements, only: :create
   resources :sources do

data/db/migrate/20260313120000_add_composite_indexes_to_log_tables.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class AddCompositeIndexesToLogTables < ActiveRecord::Migration[8.0]
+  def change
+    add_index :sourcemon_fetch_logs, [ :source_id, :started_at ],
+              name: "index_fetch_logs_on_source_id_and_started_at"
+    add_index :sourcemon_scrape_logs, [ :source_id, :started_at ],
+              name: "index_scrape_logs_on_source_id_and_started_at"
+    add_index :sourcemon_scrape_logs, [ :item_id, :started_at ],
+              name: "index_scrape_logs_on_item_id_and_started_at"
+    add_index :sourcemon_health_check_logs, [ :source_id, :started_at ],
+              name: "index_health_check_logs_on_source_id_and_started_at"
+  end
+end

data/db/migrate/20260314120000_align_health_status_default.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class AlignHealthStatusDefault < ActiveRecord::Migration[8.1]
+  def up
+    change_column_default :sourcemon_sources, :health_status, from: "healthy", to: "working"
+  end
+
+  def down
+    change_column_default :sourcemon_sources, :health_status, from: "working", to: "healthy"
+  end
+end

data/lib/source_monitor/analytics/sources_index_metrics.rb

@@ -43,6 +43,21 @@ module SourceMonitor
         @item_activity_rates ||= SourceActivityRates.new(scope: result_scope, lookback:, now:).per_source_rates
       end
 
+      def word_count_averages(source_ids)
+        if source_ids.any?
+          base = SourceMonitor::ItemContent.joins(:item).where(sourcemon_items: { source_id: source_ids })
+          feed = base.where.not(feed_word_count: nil)
+                     .group("sourcemon_items.source_id")
+                     .average(:feed_word_count)
+          scraped = base.where.not(scraped_word_count: nil)
+                       .group("sourcemon_items.source_id")
+                       .average(:scraped_word_count)
+          { feed:, scraped: }
+        else
+          { feed: {}, scraped: {} }
+        end
+      end
+
       def fetch_interval_filter
         min = integer_param(search_params["fetch_interval_minutes_gteq"])
         max = integer_param(search_params["fetch_interval_minutes_lt"]) || integer_param(search_params["fetch_interval_minutes_lteq"])

data/lib/source_monitor/dashboard/queries/recent_activity_query.rb

@@ -43,7 +43,7 @@ module SourceMonitor
         end
 
         def sanitized_sql
-          ActiveRecord::Base.send(:sanitize_sql_array, [ unified_sql_template, limit ])
+          ActiveRecord::Base.send(:sanitize_sql_array, [ unified_sql_template, limit, limit, limit, limit ])
         end
 
         def unified_sql_template
@@ -61,11 +61,11 @@ module SourceMonitor
                    source_id,
                    source_feed_url
             FROM (
-              #{fetch_log_sql}
+              (#{fetch_log_sql})
               UNION ALL
-              #{scrape_log_sql}
+              (#{scrape_log_sql})
               UNION ALL
-              #{item_sql}
+              (#{item_sql})
             ) AS dashboard_events
             WHERE occurred_at IS NOT NULL
             ORDER BY occurred_at DESC
@@ -91,6 +91,8 @@ module SourceMonitor
             FROM #{SourceMonitor::FetchLog.quoted_table_name}
             LEFT JOIN #{SourceMonitor::Source.quoted_table_name}
               ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::FetchLog.quoted_table_name}.source_id
+            ORDER BY #{SourceMonitor::FetchLog.quoted_table_name}.started_at DESC NULLS LAST
+            LIMIT ?
           SQL
         end
 
@@ -114,6 +116,8 @@ module SourceMonitor
               ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::ScrapeLog.quoted_table_name}.source_id
             LEFT JOIN #{SourceMonitor::Item.quoted_table_name}
               ON #{SourceMonitor::Item.quoted_table_name}.id = #{SourceMonitor::ScrapeLog.quoted_table_name}.item_id
+            ORDER BY #{SourceMonitor::ScrapeLog.quoted_table_name}.started_at DESC NULLS LAST
+            LIMIT ?
           SQL
         end
 
@@ -135,6 +139,8 @@ module SourceMonitor
             FROM #{SourceMonitor::Item.quoted_table_name}
             LEFT JOIN #{SourceMonitor::Source.quoted_table_name}
               ON #{SourceMonitor::Source.quoted_table_name}.id = #{SourceMonitor::Item.quoted_table_name}.source_id
+            ORDER BY #{SourceMonitor::Item.quoted_table_name}.created_at DESC NULLS LAST
+            LIMIT ?
           SQL
         end
 

data/lib/source_monitor/dashboard/turbo_broadcaster.rb

@@ -22,17 +22,33 @@ module SourceMonitor
         @item_callback ||= lambda { |_event| broadcast_dashboard_updates }
       end
 
+      STAT_CARDS = [
+        { key: "total_sources", label: "Sources", stat: :total_sources, caption: "Total registered" },
+        { key: "active_sources", label: "Active", stat: :active_sources, caption: "Fetching on schedule" },
+        { key: "failed_sources", label: "Failures", stat: :failed_sources, caption: "Require attention" },
+        { key: "total_items", label: "Items", stat: :total_items, caption: "Stored entries" },
+        { key: "fetches_today", label: "Fetches Today", stat: :fetches_today, caption: "Completed runs" }
+      ].freeze
+
       def broadcast_dashboard_updates
         return unless turbo_streams_available?
 
         queries = SourceMonitor::Dashboard::Queries.new
         url_helpers = SourceMonitor::Engine.routes.url_helpers
+        stats = queries.stats
 
-        Turbo::StreamsChannel.broadcast_replace_to(
-          STREAM_NAME,
-          target: "source_monitor_dashboard_stats",
-          html: render_partial("source_monitor/dashboard/stats", stats: queries.stats)
-        )
+        STAT_CARDS.each do |card|
+          Turbo::StreamsChannel.broadcast_replace_to(
+            STREAM_NAME,
+            target: "source_monitor_stat_#{card[:key]}",
+            html: render_partial("source_monitor/dashboard/stat_card", stat_card: {
+              key: card[:key],
+              label: card[:label],
+              value: stats[card[:stat]],
+              caption: card[:caption]
+            })
+          )
+        end
 
         Turbo::StreamsChannel.broadcast_replace_to(
           STREAM_NAME,

data/lib/source_monitor/favicons/fetcher.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Favicons
+    # Coordinates favicon fetching for a source: checks prerequisites
+    # (ActiveStorage, config, cooldown), delegates to Discoverer, and
+    # handles attachment or failure recording. Extracted from FaviconFetchJob.
+    class Fetcher
+      TRANSIENT_ERRORS = [
+        Timeout::Error, Errno::ETIMEDOUT,
+        Faraday::TimeoutError, Faraday::ConnectionFailed,
+        Net::OpenTimeout, Net::ReadTimeout
+      ].freeze
+
+      def initialize(source)
+        @source = source
+      end
+
+      def call
+        return unless defined?(ActiveStorage)
+        return unless SourceMonitor.config.favicons.enabled?
+        return if source.website_url.blank?
+        return if source.favicon.attached?
+        return if within_cooldown?
+
+        result = SourceMonitor::Favicons::Discoverer.new(source.website_url).call
+
+        if result
+          attach_favicon(result)
+        else
+          record_failed_attempt
+        end
+      rescue ActiveRecord::Deadlocked
+        raise
+      rescue *TRANSIENT_ERRORS => error
+        log_error("Transient error", error)
+        raise
+      rescue StandardError => error
+        record_failed_attempt
+        log_error("Failed", error)
+      end
+
+      private
+
+      attr_reader :source
+
+      def within_cooldown?
+        last_attempt = source.metadata&.dig("favicon_last_attempted_at")
+        return false if last_attempt.blank?
+
+        cooldown_days = SourceMonitor.config.favicons.retry_cooldown_days
+        Time.parse(last_attempt) > cooldown_days.days.ago
+      rescue ArgumentError, TypeError
+        false
+      end
+
+      def attach_favicon(result)
+        blob = ActiveStorage::Blob.create_and_upload!(
+          io: result.io,
+          filename: result.filename,
+          content_type: result.content_type
+        )
+        source.favicon.attach(blob)
+      end
+
+      def record_failed_attempt
+        metadata = (source.metadata || {}).merge(
+          "favicon_last_attempted_at" => Time.current.iso8601
+        )
+        source.update_column(:metadata, metadata)
+      rescue StandardError
+        nil
+      end
+
+      def log_error(prefix, error)
+        return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+        Rails.logger.warn(
+          "[SourceMonitor::Favicons::Fetcher] #{prefix} for source #{source&.id}: #{error.class} - #{error.message}"
+        )
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/source_monitor/fetching/cloudflare_bypass.rb

@@ -12,12 +12,15 @@ module SourceMonitor
 
       CLOUDFLARE_MARKERS = FeedFetcher::CLOUDFLARE_MARKERS
       SNIFF_LIMIT = FeedFetcher::SNIFF_LIMIT
+      DEFAULT_MAX_ATTEMPTS = 2
+      BYPASS_TIMEOUT = 10
 
-      attr_reader :response, :feed_url
+      attr_reader :response, :feed_url, :max_attempts
 
-      def initialize(response:, feed_url:)
+      def initialize(response:, feed_url:, max_attempts: DEFAULT_MAX_ATTEMPTS)
         @response = response
         @feed_url = feed_url
+        @max_attempts = max_attempts
       end
 
       def call
@@ -36,7 +39,7 @@ module SourceMonitor
       end
 
       def attempt_ua_rotation
-        USER_AGENTS.each do |ua|
+        USER_AGENTS.first(max_attempts).each do |ua|
           headers = {
             "User-Agent" => ua,
             "Cache-Control" => "no-cache",
@@ -50,9 +53,15 @@ module SourceMonitor
       end
 
       def fetch_with_headers(headers)
-        client = SourceMonitor::HTTP.client(headers: headers, retry_requests: false)
+        client = SourceMonitor::HTTP.client(
+          headers: headers,
+          timeout: BYPASS_TIMEOUT,
+          open_timeout: [ BYPASS_TIMEOUT / 2, 5 ].min,
+          retry_requests: false
+        )
         client.get(feed_url)
-      rescue StandardError
+      rescue StandardError => e
+        Rails.logger.warn("[SourceMonitor] CloudflareBypass request failed for #{feed_url}: #{e.class}: #{e.message}") if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
         nil
       end
 

data/lib/source_monitor/fetching/completion/event_publisher.rb

@@ -5,12 +5,24 @@ module SourceMonitor
     module Completion
       # Publishes fetch completion events to the configured event dispatcher.
       class EventPublisher
+        Result = Struct.new(:status, :error, keyword_init: true) do
+          def success?
+            status != :failed
+          end
+        end
+
         def initialize(dispatcher: SourceMonitor::Events)
           @dispatcher = dispatcher
         end
 
         def call(source:, result:)
           dispatcher.after_fetch_completed(source: source, result: result)
+          Result.new(status: :published)
+        rescue StandardError => error
+          Rails.logger.error(
+            "[SourceMonitor::Fetching::Completion::EventPublisher] Event dispatch failed for source #{source.id}: #{error.class} - #{error.message}"
+          ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+          Result.new(status: :failed, error: error)
         end
 
         private

data/lib/source_monitor/fetching/completion/follow_up_handler.rb

@@ -5,25 +5,38 @@ module SourceMonitor
     module Completion
       # Enqueues follow-up scraping work for items created during a fetch.
       class FollowUpHandler
+        Result = Struct.new(:status, :enqueued_count, :errors, keyword_init: true) do
+          def success?
+            status != :failed
+          end
+        end
+
         def initialize(enqueuer_class: SourceMonitor::Scraping::Enqueuer, job_class: SourceMonitor::ScrapeItemJob)
           @enqueuer_class = enqueuer_class
           @job_class = job_class
         end
 
         def call(source:, result:)
-          return unless should_enqueue?(source:, result:)
+          return Result.new(status: :skipped, enqueued_count: 0, errors: []) unless should_enqueue?(source:, result:)
+
+          enqueued = 0
+          errors = []
 
           Array(result.item_processing&.created_items).each do |item|
             next unless item.present? && item.scraped_at.nil?
 
             begin
               enqueuer_class.enqueue(item:, source:, job_class:, reason: :auto)
+              enqueued += 1
             rescue StandardError => error
+              errors << error
               Rails.logger.error(
-                "[SourceMonitor] FollowUpHandler: failed to enqueue scrape for item #{item.id}: #{error.class}: #{error.message}"
+                "[SourceMonitor::Fetching::Completion::FollowUpHandler] Failed to enqueue scrape for item #{item.id}: #{error.class}: #{error.message}"
               ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
             end
           end
+
+          Result.new(status: :applied, enqueued_count: enqueued, errors: errors)
         end
 
         private

data/lib/source_monitor/fetching/completion/retention_handler.rb

@@ -5,20 +5,28 @@ module SourceMonitor
     module Completion
       # Applies item retention after a fetch completes.
       class RetentionHandler
+        Result = Struct.new(:status, :removed_total, :error, keyword_init: true) do
+          def success?
+            status != :failed
+          end
+        end
+
         def initialize(pruner: SourceMonitor::Items::RetentionPruner)
           @pruner = pruner
         end
 
         def call(source:, result:) # rubocop:disable Lint/UnusedMethodArgument
-          pruner.call(
+          pruner_result = pruner.call(
             source: source,
             strategy: SourceMonitor.config.retention.strategy
           )
+          removed = pruner_result.respond_to?(:removed_total) ? pruner_result.removed_total : 0
+          Result.new(status: :applied, removed_total: removed)
         rescue StandardError => error
           Rails.logger.error(
-            "[SourceMonitor] Retention pruning failed for source #{source.id}: #{error.class} - #{error.message}"
+            "[SourceMonitor::Fetching::Completion::RetentionHandler] Retention pruning failed for source #{source.id}: #{error.class} - #{error.message}"
           )
-          nil
+          Result.new(status: :failed, removed_total: 0, error: error)
         end
 
         private

data/lib/source_monitor/fetching/feed_fetcher.rb

@@ -27,13 +27,6 @@ module SourceMonitor
       )
       ResponseWrapper = Struct.new(:status, :headers, :body, keyword_init: true)
 
-      MIN_FETCH_INTERVAL = AdaptiveInterval::MIN_FETCH_INTERVAL
-      MAX_FETCH_INTERVAL = AdaptiveInterval::MAX_FETCH_INTERVAL
-      INCREASE_FACTOR = AdaptiveInterval::INCREASE_FACTOR
-      DECREASE_FACTOR = AdaptiveInterval::DECREASE_FACTOR
-      FAILURE_INCREASE_FACTOR = AdaptiveInterval::FAILURE_INCREASE_FACTOR
-      JITTER_PERCENT = AdaptiveInterval::JITTER_PERCENT
-
       attr_reader :source, :client, :jitter_proc
 
       def initialize(source:, client: nil, jitter: nil)
@@ -328,7 +321,8 @@ module SourceMonitor
 
         response = perform_request
         handle_response(response, started_at, instrumentation_payload)
-      rescue StandardError
+      rescue StandardError => e
+        Rails.logger.warn("[SourceMonitor] AIA recovery failed for #{source.feed_url}: #{e.class}: #{e.message}") if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
         nil
       end
 
@@ -389,19 +383,6 @@ module SourceMonitor
       def entry_processor
         @entry_processor ||= EntryProcessor.new(source: source)
       end
-
-      # Forwarding methods for backward compatibility with tests
-      def process_feed_entries(feed) = entry_processor.process_feed_entries(feed)
-      def jitter_offset(interval_seconds) = adaptive_interval.jitter_offset(interval_seconds)
-      def adjusted_interval_with_jitter(interval_seconds) = adaptive_interval.adjusted_interval_with_jitter(interval_seconds)
-      def updated_metadata(feed_signature: nil) = source_updater.updated_metadata(feed_signature: feed_signature)
-      def feed_signature_changed?(feed_signature) = source_updater.feed_signature_changed?(feed_signature)
-      def configured_seconds(minutes_value, default) = adaptive_interval.configured_seconds(minutes_value, default)
-      def configured_positive(value, default) = adaptive_interval.configured_positive(value, default)
-      def configured_non_negative(value, default) = adaptive_interval.configured_non_negative(value, default)
-      def interval_minutes_for(interval_seconds) = adaptive_interval.interval_minutes_for(interval_seconds)
-      def parse_http_time(value) = source_updater.parse_http_time(value)
-      def extract_numeric(value) = adaptive_interval.extract_numeric(value)
     end
   end
 end

data/lib/source_monitor/fetching/fetch_runner.rb

@@ -59,13 +59,13 @@ module SourceMonitor
         lock.with_lock do
           mark_fetching!
           result = fetcher_class.new(source: source).call
-          retention_handler.call(source:, result:)
-          follow_up_handler.call(source:, result:)
+          log_handler_result("RetentionHandler", retention_handler.call(source:, result:))
+          log_handler_result("FollowUpHandler", follow_up_handler.call(source:, result:))
           schedule_retry_if_needed(result)
           mark_complete!(result)
         end
 
-        event_publisher.call(source:, result:)
+        log_handler_result("EventPublisher", event_publisher.call(source:, result:))
         result
       rescue SourceMonitor::Fetching::AdvisoryLock::NotAcquiredError => error
         raise ConcurrencyError, error.message
@@ -138,6 +138,15 @@ module SourceMonitor
         self.class.send(:update_source_state!, source, attrs)
       end
 
+      def log_handler_result(handler_name, handler_result)
+        return unless handler_result.respond_to?(:success?) && !handler_result.success?
+
+        error_detail = handler_result.respond_to?(:error) && handler_result.error ? ": #{handler_result.error.message}" : ""
+        Rails.logger.warn(
+          "[SourceMonitor::Fetching::FetchRunner] #{handler_name} failed for source #{source.id}#{error_detail}"
+        ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+      end
+
       def schedule_retry_if_needed(result)
         decision = result&.retry_decision
         return unless decision&.retry?

data/lib/source_monitor/fetching/retry_orchestrator.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Fetching
+    # Executes retry/circuit-breaker decisions produced by RetryPolicy.
+    #
+    # Accepts a source, the original fetch error, and a RetryPolicy::Decision,
+    # then either enqueues a retry job, opens the circuit, or resets retry state.
+    #
+    # Returns a Result struct indicating which path was taken.
+    class RetryOrchestrator
+      Result = Struct.new(:status, :source, :error, :decision, keyword_init: true) do
+        def retry_enqueued?
+          status == :retry_enqueued
+        end
+
+        def circuit_opened?
+          status == :circuit_opened
+        end
+
+        def exhausted?
+          status == :exhausted
+        end
+      end
+
+      def self.call(source:, error:, decision:, job_class: SourceMonitor::FetchFeedJob, now: Time.current)
+        new(source:, error:, decision:, job_class:, now:).call
+      end
+
+      def initialize(source:, error:, decision:, job_class:, now:)
+        @source = source
+        @error = error
+        @decision = decision
+        @job_class = job_class
+        @now = now
+      end
+
+      def call
+        if decision.retry?
+          enqueue_retry!
+        elsif decision.open_circuit?
+          open_circuit!
+        else
+          reset_retry_state!
+        end
+      end
+
+      private
+
+      attr_reader :source, :error, :decision, :job_class, :now
+
+      def enqueue_retry!
+        retry_at = now + (decision.wait || 0)
+
+        source.with_lock do
+          source.reload
+          source.update!(
+            fetch_retry_attempt: decision.next_attempt,
+            fetch_circuit_opened_at: nil,
+            fetch_circuit_until: nil,
+            next_fetch_at: retry_at,
+            backoff_until: retry_at,
+            fetch_status: "queued"
+          )
+        end
+
+        job_class.set(wait: decision.wait || 0).perform_later(source.id)
+
+        Result.new(status: :retry_enqueued, source: source, error: error, decision: decision)
+      end
+
+      def open_circuit!
+        source.with_lock do
+          source.reload
+          source.update!(
+            fetch_retry_attempt: 0,
+            fetch_circuit_opened_at: now,
+            fetch_circuit_until: decision.circuit_until,
+            next_fetch_at: decision.circuit_until,
+            backoff_until: decision.circuit_until,
+            fetch_status: "failed"
+          )
+        end
+
+        Result.new(status: :circuit_opened, source: source, error: error, decision: decision)
+      end
+
+      def reset_retry_state!
+        source.with_lock do
+          source.reload
+          source.update!(
+            fetch_retry_attempt: 0,
+            fetch_circuit_opened_at: nil,
+            fetch_circuit_until: nil
+          )
+        end
+
+        Result.new(status: :exhausted, source: source, error: error, decision: decision)
+      end
+    end
+  end
+end

data/lib/source_monitor/fetching/stalled_fetch_reconciler.rb

@@ -102,6 +102,15 @@ module SourceMonitor
         return ::SolidQueue::Job.none unless jobs_supported?
 
         queue_name = SourceMonitor.queue_name(:fetch)
+        # SolidQueue stores job arguments as JSON in the `arguments` text column.
+        # The format is: {"job_class":"...", "arguments":[source_id, ...], ...}
+        # We cast to jsonb and extract the first positional argument to match
+        # jobs targeting this source.
+        #
+        # Tested against: SolidQueue 1.1.x (Rails 8.x). The serialization format
+        # is part of ActiveJob's serialize/deserialize contract. If SolidQueue
+        # changes its storage format, this query will silently return no matches
+        # (safe failure). Re-verify on SolidQueue major version upgrades.
         ::SolidQueue::Job.
           where(queue_name: queue_name).
           where("arguments::jsonb -> 'arguments' ->> 0 = ?", source.id.to_s)