source_monitor 0.11.1 → 0.12.0

data/lib/source_monitor/health/source_health_check_orchestrator.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Health
+    # Orchestrates a source health check: runs the probe, broadcasts
+    # UI updates with toast notifications, triggers a follow-up fetch
+    # for degraded sources, and handles unexpected errors gracefully.
+    # Extracted from SourceHealthCheckJob.
+    class SourceHealthCheckOrchestrator
+      DEGRADED_STATUSES = %w[declining failing].freeze
+
+      def initialize(source)
+        @source = source
+      end
+
+      def call
+        result = SourceMonitor::Health::SourceHealthCheck.new(source: source).call
+        broadcast_outcome(result)
+        trigger_fetch_if_degraded(result)
+      rescue StandardError => error
+        log_error(error)
+        record_unexpected_failure(error)
+        broadcast_outcome(nil, error)
+      end
+
+      private
+
+      attr_reader :source
+
+      def trigger_fetch_if_degraded(result)
+        return unless result&.success?
+        return unless DEGRADED_STATUSES.include?(source.health_status.to_s)
+
+        SourceMonitor::FetchFeedJob.perform_later(source.id, force: true)
+      end
+
+      def record_unexpected_failure(error)
+        SourceMonitor::HealthCheckLog.create!(
+          source: source,
+          success: false,
+          started_at: Time.current,
+          completed_at: Time.current,
+          duration_ms: 0,
+          error_class: error.class.name,
+          error_message: error.message
+        )
+      rescue StandardError
+        nil
+      end
+
+      def broadcast_outcome(result, error = nil)
+        SourceMonitor::Realtime.broadcast_source(source)
+
+        message, level = toast_payload(result, error)
+        return if message.blank?
+
+        SourceMonitor::Realtime.broadcast_toast(message: message, level: level)
+      end
+
+      def toast_payload(result, error)
+        if error
+          return [
+            "Health check failed for #{source.name}: #{error.message}",
+            :error
+          ]
+        end
+
+        if result&.success?
+          [
+            "Health check succeeded for #{source.name}.",
+            :success
+          ]
+        else
+          failure_reason = result&.error&.message
+          http_status = result&.log&.http_status
+          message = "Health check failed for #{source.name}"
+          message += " (HTTP #{http_status})" if http_status.present?
+          message += ": #{failure_reason}" if failure_reason.present?
+          [
+            "#{message}.",
+            :error
+          ]
+        end
+      end
+
+      def log_error(error)
+        return unless defined?(Rails) && Rails.respond_to?(:logger)
+
+        Rails.logger&.error(
+          "[SourceMonitor::Health::SourceHealthCheckOrchestrator] error for source #{source.id}: #{error.class}: #{error.message}"
+        )
+      end
+    end
+  end
+end

data/lib/source_monitor/health.rb

@@ -3,6 +3,7 @@
 require "source_monitor/health/source_health_monitor"
 require "source_monitor/health/source_health_reset"
 require "source_monitor/health/source_health_check"
+require "source_monitor/health/source_health_check_orchestrator"
 require "source_monitor/health/import_source_health_check"
 
 module SourceMonitor

data/lib/source_monitor/images/downloader.rb

@@ -43,13 +43,12 @@ module SourceMonitor
       private
 
       def fetch_image
-        connection = Faraday.new do |f|
-          f.options.timeout = settings.download_timeout
-          f.options.open_timeout = [ settings.download_timeout / 2, 5 ].min
-          f.headers["User-Agent"] = SourceMonitor.config.http.user_agent || "SourceMonitor/#{SourceMonitor::VERSION}"
-          f.headers["Accept"] = "image/*"
-          f.adapter Faraday.default_adapter
-        end
+        connection = SourceMonitor::HTTP.client(
+          timeout: settings.download_timeout,
+          open_timeout: [ settings.download_timeout / 2, 5 ].min,
+          headers: { "Accept" => "image/*" },
+          retry_requests: false
+        )
 
         response = connection.get(url)
         return response if response.status == 200

data/lib/source_monitor/images/processor.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "faraday"
+
+module SourceMonitor
+  module Images
+    # Orchestrates downloading images from an item's HTML content, attaching
+    # them via ActiveStorage, and rewriting the HTML to use local blob URLs.
+    # Extracted from DownloadContentImagesJob for testability and reuse.
+    class Processor
+      TRANSIENT_ERRORS = [
+        Timeout::Error, Errno::ETIMEDOUT,
+        Faraday::TimeoutError, Faraday::ConnectionFailed,
+        Net::OpenTimeout, Net::ReadTimeout
+      ].freeze
+
+      def initialize(item)
+        @item = item
+      end
+
+      def call
+        return unless SourceMonitor.config.images.download_enabled?
+
+        html = item.content
+        return if html.blank?
+
+        item_content = item.item_content || item.build_item_content
+
+        # Skip if images already attached (idempotency)
+        return if item_content.persisted? && item_content.images.attached?
+
+        base_url = item.url
+        rewriter = SourceMonitor::Images::ContentRewriter.new(html, base_url: base_url)
+        image_urls = rewriter.image_urls
+        return if image_urls.empty?
+
+        # Save item_content first so we can attach blobs to it
+        item_content.save! unless item_content.persisted?
+
+        # Download images and build URL mapping
+        url_mapping = download_images(item_content, image_urls)
+        return if url_mapping.empty?
+
+        # Rewrite HTML with Active Storage URLs
+        rewritten_html = rewriter.rewrite do |original_url|
+          url_mapping[original_url]
+        end
+
+        # Update the item content with rewritten HTML
+        item.update!(content: rewritten_html)
+      end
+
+      private
+
+      attr_reader :item
+
+      def download_images(item_content, image_urls)
+        url_mapping = {}
+        settings = SourceMonitor.config.images
+
+        image_urls.each do |image_url|
+          result = SourceMonitor::Images::Downloader.new(image_url, settings: settings).call
+          next unless result
+
+          blob = ActiveStorage::Blob.create_and_upload!(
+            io: result.io,
+            filename: result.filename,
+            content_type: result.content_type
+          )
+          item_content.images.attach(blob)
+
+          url_mapping[image_url] = Rails.application.routes.url_helpers.rails_blob_path(blob, only_path: true)
+        rescue ActiveRecord::Deadlocked
+          raise # let job framework retry on database deadlock
+        rescue *TRANSIENT_ERRORS
+          raise # re-raise transient errors to abort job for framework retry
+        rescue StandardError => error
+          # Individual image failure should not block others.
+          # Original URL will be preserved (graceful fallback).
+          log_image_error(image_url, error)
+          next
+        end
+
+        url_mapping
+      end
+
+      def log_image_error(image_url, error)
+        return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+        Rails.logger.warn(
+          "[SourceMonitor::Images::Processor] Skipping image #{image_url}: #{error.class} - #{error.message}"
+        )
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/source_monitor/import_sessions/health_check_updater.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require "source_monitor/health/import_source_health_check"
+require "source_monitor/import_sessions/entry_normalizer"
+require "source_monitor/import_sessions/health_check_broadcaster"
+
+module SourceMonitor
+  module ImportSessions
+    # Performs a health check for a single OPML import entry: probes the feed
+    # URL, acquires a row lock to merge the result into the import session,
+    # and broadcasts UI updates. Extracted from ImportSessionHealthCheckJob.
+    class HealthCheckUpdater
+      def initialize(import_session:, entry_id:)
+        @import_session = import_session
+        @entry_id = entry_id
+      end
+
+      def call
+        return unless active_for?(import_session)
+
+        result = perform_health_check
+        return unless result
+
+        updated_entry = nil
+
+        import_session.with_lock do
+          import_session.reload
+          return unless active_for?(import_session)
+
+          entries = Array(import_session.parsed_sources).map(&:to_h)
+          index = entries.index { |candidate| entry_id_for(candidate) == entry_id.to_s }
+          return unless index
+
+          entries[index] = entries[index].merge(
+            "health_status" => result.status,
+            "health_error" => result.error_message
+          )
+
+          selected_ids = Array(import_session.selected_source_ids).map(&:to_s)
+          selected_ids -= [ entry_id.to_s ] if result.status == "unhealthy"
+
+          attrs = {
+            parsed_sources: entries,
+            selected_source_ids: selected_ids,
+            health_check_completed_at: completion_time(entries, import_session.health_check_targets)
+          }.compact
+
+          import_session.update!(attrs)
+          normalized_entry = SourceMonitor::ImportSessions::EntryNormalizer.normalize(entries[index])
+          updated_entry = normalized_entry.merge(selected: selected_ids.include?(entry_id.to_s))
+        end
+
+        broadcaster = SourceMonitor::ImportSessions::HealthCheckBroadcaster.new(import_session)
+        broadcaster.broadcast_row(updated_entry) if updated_entry
+        broadcaster.broadcast_progress
+      end
+
+      private
+
+      attr_reader :import_session, :entry_id
+
+      def active_for?(session)
+        session.current_step == "health_check" && session.health_checks_active?
+      end
+
+      def perform_health_check
+        entry = find_entry
+        return unless entry
+
+        SourceMonitor::Health::ImportSourceHealthCheck.new(feed_url: entry_feed_url(entry)).call
+      end
+
+      def find_entry
+        Array(import_session.parsed_sources).find { |entry| entry_id_for(entry) == entry_id.to_s }
+      end
+
+      def entry_id_for(entry)
+        entry.to_h["id"].presence || entry.to_h[:id].presence || entry.to_h["feed_url"].to_s
+      end
+
+      def entry_feed_url(entry)
+        entry.to_h["feed_url"] || entry.to_h[:feed_url]
+      end
+
+      def completion_time(entries, targets)
+        normalized = Array(entries).map { |entry| SourceMonitor::ImportSessions::EntryNormalizer.normalize(entry) }
+        filtered = normalized.select { |entry| targets.include?(entry[:id]) }
+        return nil if filtered.empty?
+
+        completed = filtered.count { |entry| %w[healthy unhealthy].include?(entry[:health_status].to_s) }
+        completed >= filtered.size ? Time.current : nil
+      end
+    end
+  end
+end

data/lib/source_monitor/import_sessions/opml_importer.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require "set"
+require "source_monitor/import_sessions/entry_normalizer"
+require "source_monitor/realtime/broadcaster"
+require "source_monitor/sources/params"
+
+module SourceMonitor
+  module ImportSessions
+    # Orchestrates OPML import: selects entries, deduplicates, creates sources,
+    # records results, and broadcasts completion. Extracted from ImportOpmlJob
+    # so import logic can be invoked synchronously (console, tests).
+    class OPMLImporter
+      def initialize(import_session:, import_history:)
+        @import_session = import_session
+        @import_history = import_history
+      end
+
+      def call
+        import_history.update_columns(started_at: Time.current) unless import_history.started_at
+
+        processed = Set.new
+
+        selected_entries.each do |entry|
+          process_entry(entry, processed)
+        end
+
+        import_history.update!(
+          imported_sources: imported_sources,
+          failed_sources: failed_sources,
+          skipped_duplicates: skipped_duplicates,
+          bulk_settings: import_session.bulk_settings.presence || {},
+          completed_at: Time.current
+        )
+
+        broadcast_completion(import_history)
+      end
+
+      private
+
+      attr_reader :import_session, :import_history
+
+      def selected_entries
+        ids = Array(import_session.selected_source_ids).map(&:to_s)
+
+        Array(import_session.parsed_sources)
+          .map { |entry| SourceMonitor::ImportSessions::EntryNormalizer.normalize(entry) }
+          .select { |entry| ids.include?(entry[:id]) }
+      end
+
+      def process_entry(entry, processed)
+        feed_url = entry[:feed_url].to_s
+
+        if feed_url.blank?
+          failed_sources << failure_payload(feed_url, "MissingFeedURL", "Feed URL is missing")
+          return
+        end
+
+        normalized_url = feed_url.downcase
+
+        if processed.include?(normalized_url)
+          skipped_duplicates << skipped_payload(feed_url, "duplicate in import selection")
+          return
+        end
+
+        if duplicate_source?(normalized_url)
+          skipped_duplicates << skipped_payload(feed_url, "already exists")
+          processed << normalized_url
+          return
+        end
+
+        source = SourceMonitor::Source.new(build_attributes(entry))
+
+        if source.save
+          imported_sources << { id: source.id, feed_url: source.feed_url, name: source.name }
+          SourceMonitor::FaviconFetchJob.perform_later(source.id) if should_fetch_favicon?(source)
+          processed << normalized_url
+        else
+          failed_sources << failure_payload(feed_url, "ValidationFailed", source.errors.full_messages.to_sentence)
+        end
+      rescue ActiveRecord::RecordNotUnique
+        skipped_duplicates << skipped_payload(feed_url, "already exists")
+        processed << normalized_url
+      rescue StandardError => error
+        failed_sources << failure_payload(feed_url, error.class.name, error.message)
+      end
+
+      def duplicate_source?(normalized_feed_url)
+        SourceMonitor::Source.where("LOWER(feed_url) = ?", normalized_feed_url).exists?
+      end
+
+      def build_attributes(entry)
+        defaults = SourceMonitor::Sources::Params.default_attributes.deep_dup
+        settings = SourceMonitor::Security::ParameterSanitizer.sanitize(import_session.bulk_settings.presence || {})
+        settings = settings.deep_symbolize_keys
+
+        defaults.merge(settings).merge(identity_attributes(entry))
+      end
+
+      def identity_attributes(entry)
+        {
+          name: entry[:title].presence || entry[:feed_url],
+          feed_url: entry[:feed_url],
+          website_url: entry[:website_url]
+        }
+      end
+
+      def imported_sources
+        @imported_sources ||= []
+      end
+
+      def failed_sources
+        @failed_sources ||= []
+      end
+
+      def skipped_duplicates
+        @skipped_duplicates ||= []
+      end
+
+      def failure_payload(feed_url, error_class, message)
+        {
+          feed_url: feed_url,
+          error_class: error_class,
+          error_message: message
+        }
+      end
+
+      def skipped_payload(feed_url, reason)
+        {
+          feed_url: feed_url,
+          reason: reason
+        }
+      end
+
+      def should_fetch_favicon?(source)
+        defined?(ActiveStorage) &&
+          SourceMonitor.config.favicons.enabled? &&
+          source.website_url.present?
+      rescue StandardError
+        false
+      end
+
+      def broadcast_completion(history)
+        return unless defined?(Turbo::StreamsChannel)
+
+        histories = SourceMonitor::ImportHistory.recent_for(history.user_id).limit(5)
+
+        Turbo::StreamsChannel.broadcast_replace_to(
+          SourceMonitor::Realtime::Broadcaster::SOURCE_INDEX_STREAM,
+          target: "source_monitor_import_history_panel",
+          html: SourceMonitor::SourcesController.render(
+            partial: "source_monitor/sources/import_history_panel",
+            locals: { import_histories: histories }
+          )
+        )
+      rescue StandardError => error
+        if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+          Rails.logger.error("[SourceMonitor::ImportSessions::OPMLImporter] broadcast failed: #{error.class}: #{error.message}")
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/items/item_creator.rb

@@ -120,7 +120,6 @@ module SourceMonitor
         new_item = SourceMonitor::Item.new(source_id: source.id)
         apply_attributes(new_item, attributes)
         new_item.save!
-        new_item.ensure_feed_content_record
         Result.new(item: new_item, status: :created)
       rescue ActiveRecord::RecordNotUnique
         handle_concurrent_duplicate(attributes, raw_guid_present:)
@@ -176,26 +175,6 @@ module SourceMonitor
       def content_extractor
         @content_extractor ||= ContentExtractor.new(source: source)
       end
-
-      # Forwarding methods for backward compatibility with tests
-      def process_feed_content(raw_content, title:) = content_extractor.process_feed_content(raw_content, title: title)
-      def should_process_feed_content?(raw_content) = content_extractor.should_process_feed_content?(raw_content)
-      def feed_content_parser_class = content_extractor.feed_content_parser_class
-      def wrap_content_for_readability(content, title:) = content_extractor.wrap_content_for_readability(content, title: title)
-      def default_feed_readability_options = content_extractor.default_feed_readability_options
-      def build_feed_content_metadata(result:, raw_content:, processed_content:)
-        content_extractor.build_feed_content_metadata(result: result, raw_content: raw_content, processed_content: processed_content)
-      end
-      def html_fragment?(value) = content_extractor.html_fragment?(value)
-      def deep_copy(value) = content_extractor.deep_copy(value)
-      def string_or_nil(value) = entry_parser.string_or_nil(value)
-      def sanitize_string_array(values) = entry_parser.sanitize_string_array(values)
-      def split_keywords(value) = entry_parser.split_keywords(value)
-      def safe_integer(value) = entry_parser.safe_integer(value)
-      def json_entry? = entry_parser.json_entry?
-      def atom_entry? = entry_parser.atom_entry?
-      def normalize_metadata(value) = entry_parser.normalize_metadata(value)
-      def generate_fingerprint(title, url, content) = entry_parser.generate_fingerprint(title, url, content)
     end
   end
 end

data/lib/source_monitor/logs/query.rb

@@ -9,6 +9,7 @@ module SourceMonitor
         :per_page,
         :has_next_page,
         :has_previous_page,
+        :total_count,
         :filter_set,
         keyword_init: true
       ) do
@@ -19,6 +20,24 @@ module SourceMonitor
         def has_previous_page?
           !!self[:has_previous_page]
         end
+
+        def next_page
+          return nil unless has_next_page?
+
+          page + 1
+        end
+
+        def previous_page
+          return nil unless has_previous_page?
+
+          [ page - 1, 1 ].max
+        end
+
+        def total_pages
+          return 1 if total_count.nil? || total_count <= 0
+
+          [ 1, (total_count.to_f / per_page).ceil ].max
+        end
       end
 
       def initialize(params:)
@@ -38,6 +57,7 @@ module SourceMonitor
           per_page: pagination_result.per_page,
           has_next_page: pagination_result.has_next_page?,
           has_previous_page: pagination_result.has_previous_page?,
+          total_count: pagination_result.total_count,
           filter_set:
         )
       end

data/lib/source_monitor/queries/scrape_candidates_query.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Queries
+    class ScrapeCandidatesQuery
+      def initialize(threshold: SourceMonitor.config.scraping.scrape_recommendation_threshold)
+        @threshold = threshold.to_i
+      end
+
+      def call
+        return SourceMonitor::Source.none if @threshold <= 0
+
+        SourceMonitor::Source.active
+          .where(scraping_enabled: false)
+          .where(id: source_ids_below_threshold)
+      end
+
+      private
+
+      def source_ids_below_threshold
+        SourceMonitor::Item
+          .joins(:item_content)
+          .where.not(SourceMonitor::ItemContent.table_name => { feed_word_count: nil })
+          .group(:source_id)
+          .having("AVG(#{SourceMonitor::ItemContent.table_name}.feed_word_count) < ?", @threshold)
+          .select(:source_id)
+      end
+    end
+  end
+end

data/lib/source_monitor/queries.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Queries
+    autoload :ScrapeCandidatesQuery, "source_monitor/queries/scrape_candidates_query"
+  end
+end

data/lib/source_monitor/scheduler.rb

@@ -37,6 +37,11 @@ module SourceMonitor
 
         source_ids.size
       end
+    rescue StandardError => error
+      Rails.logger.warn(
+        "[SourceMonitor::Scheduler] Scheduler run failed: #{error.class} - #{error.message}"
+      ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+      0
     end
 
     private

data/lib/source_monitor/scraping/bulk_result_presenter.rb

@@ -5,11 +5,10 @@ module SourceMonitor
     # Presenter for building flash messages from BulkSourceScraper results
     # Extracts complex message formatting logic from the controller
     class BulkResultPresenter
-      attr_reader :result, :pluralizer
+      attr_reader :result
 
-      def initialize(result:, pluralizer:)
+      def initialize(result:)
         @result = result
-        @pluralizer = pluralizer
       end
 
       def to_flash_payload
@@ -25,14 +24,18 @@ module SourceMonitor
 
       private
 
+      def pluralize(count, word)
+        "#{count} #{count == 1 ? word : word.pluralize}"
+      end
+
       def build_success_payload
         label = BulkSourceScraper.selection_label(result.selection)
-        pluralized_enqueued = pluralizer.call(result.enqueued_count, "item")
+        pluralized_enqueued = pluralize(result.enqueued_count, "item")
 
         message = "Queued scraping for #{pluralized_enqueued} from the #{label}."
 
         if result.already_enqueued_count.positive?
-          pluralized_already = pluralizer.call(result.already_enqueued_count, "item")
+          pluralized_already = pluralize(result.already_enqueued_count, "item")
           message = "#{message} #{pluralized_already.capitalize} already in progress."
         end
 
@@ -44,12 +47,12 @@ module SourceMonitor
         parts = []
 
         if result.enqueued_count.positive?
-          pluralized_enqueued = pluralizer.call(result.enqueued_count, "item")
+          pluralized_enqueued = pluralize(result.enqueued_count, "item")
           parts << "Queued #{pluralized_enqueued} from the #{label}"
         end
 
         if result.already_enqueued_count.positive?
-          pluralized_already = pluralizer.call(result.already_enqueued_count, "item")
+          pluralized_already = pluralize(result.already_enqueued_count, "item")
           parts << "#{pluralized_already.capitalize} already in progress"
         end
 
@@ -62,7 +65,7 @@ module SourceMonitor
         if other_failures.values.sum.positive?
           skipped = other_failures.map do |status, count|
             label_key = status.to_s.tr("_", " ")
-            "#{pluralizer.call(count, label_key)}"
+            "#{pluralize(count, label_key)}"
           end.join(", ")
           parts << "Skipped #{skipped}"
         end