source_monitor 0.11.1 → 0.12.0

data/app/jobs/source_monitor/download_content_images_job.rb

@@ -9,66 +9,8 @@ module SourceMonitor
     def perform(item_id)
       item = SourceMonitor::Item.find_by(id: item_id)
       return unless item
-      return unless SourceMonitor.config.images.download_enabled?
 
-      html = item.content
-      return if html.blank?
-
-      # Build or find item_content for attachment storage
-      item_content = item.item_content || item.build_item_content
-
-      # Skip if images already attached (idempotency)
-      return if item_content.persisted? && item_content.images.attached?
-
-      base_url = item.url
-      rewriter = SourceMonitor::Images::ContentRewriter.new(html, base_url: base_url)
-      image_urls = rewriter.image_urls
-      return if image_urls.empty?
-
-      # Save item_content first so we can attach blobs to it
-      item_content.save! unless item_content.persisted?
-
-      # Download images and build URL mapping
-      url_mapping = download_images(item_content, image_urls)
-      return if url_mapping.empty?
-
-      # Rewrite HTML with Active Storage URLs
-      rewritten_html = rewriter.rewrite do |original_url|
-        url_mapping[original_url]
-      end
-
-      # Update the item content with rewritten HTML
-      item.update!(content: rewritten_html)
-    end
-
-    private
-
-    def download_images(item_content, image_urls)
-      url_mapping = {}
-      settings = SourceMonitor.config.images
-
-      image_urls.each do |image_url|
-        result = SourceMonitor::Images::Downloader.new(image_url, settings: settings).call
-        next unless result
-
-        blob = ActiveStorage::Blob.create_and_upload!(
-          io: result.io,
-          filename: result.filename,
-          content_type: result.content_type
-        )
-        item_content.images.attach(blob)
-
-        # Generate a serving URL for the blob
-        url_mapping[image_url] = Rails.application.routes.url_helpers.rails_blob_path(blob, only_path: true)
-      rescue ActiveRecord::Deadlocked
-        raise # let job framework retry on database deadlock
-      rescue StandardError
-        # Individual image failure should not block others.
-        # Original URL will be preserved (graceful fallback).
-        next
-      end
-
-      url_mapping
+      SourceMonitor::Images::Processor.new(item).call
     end
   end
 end

data/app/jobs/source_monitor/favicon_fetch_job.rb

@@ -7,67 +7,10 @@ module SourceMonitor
     discard_on ActiveJob::DeserializationError
 
     def perform(source_id)
-      return unless defined?(ActiveStorage)
-
       source = SourceMonitor::Source.find_by(id: source_id)
       return unless source
-      return unless SourceMonitor.config.favicons.enabled?
-      return if source.website_url.blank?
-      return if source.favicon.attached?
-      return if within_cooldown?(source)
-
-      result = SourceMonitor::Favicons::Discoverer.new(source.website_url).call
-
-      if result
-        attach_favicon(source, result)
-      else
-        record_failed_attempt(source)
-      end
-    rescue ActiveRecord::Deadlocked
-      raise # let job framework retry on database deadlock
-    rescue StandardError => error
-      record_failed_attempt(source) if source
-      log_error(source, error)
-    end
-
-    private
-
-    def within_cooldown?(source)
-      last_attempt = source.metadata&.dig("favicon_last_attempted_at")
-      return false if last_attempt.blank?
-
-      cooldown_days = SourceMonitor.config.favicons.retry_cooldown_days
-      Time.parse(last_attempt) > cooldown_days.days.ago
-    rescue ArgumentError, TypeError
-      false
-    end
-
-    def attach_favicon(source, result)
-      blob = ActiveStorage::Blob.create_and_upload!(
-        io: result.io,
-        filename: result.filename,
-        content_type: result.content_type
-      )
-      source.favicon.attach(blob)
-    end
-
-    def record_failed_attempt(source)
-      metadata = (source.metadata || {}).merge(
-        "favicon_last_attempted_at" => Time.current.iso8601
-      )
-      source.update_column(:metadata, metadata)
-    rescue StandardError
-      nil
-    end
-
-    def log_error(source, error)
-      return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
 
-      Rails.logger.warn(
-        "[SourceMonitor::FaviconFetchJob] Failed for source #{source&.id}: #{error.class} - #{error.message}"
-      )
-    rescue StandardError
-      nil
+      SourceMonitor::Favicons::Fetcher.new(source).call
     end
   end
 end

data/app/jobs/source_monitor/fetch_feed_job.rb

@@ -72,63 +72,13 @@ module SourceMonitor
       decision = SourceMonitor::Fetching::RetryPolicy.new(source:, error:, now: Time.current).decision
       return raise error unless decision
 
-      if decision.retry?
-        enqueue_retry!(source, decision)
-      elsif decision.open_circuit?
-        open_circuit!(source, decision)
-        raise error
-      else
-        reset_retry_state!(source)
-        raise error
-      end
+      result = SourceMonitor::Fetching::RetryOrchestrator.call(source:, error:, decision:)
+      raise error unless result.retry_enqueued?
     rescue StandardError => policy_error
       log_retry_failure(source, error, policy_error)
       raise error
     end
 
-    def enqueue_retry!(source, decision)
-      retry_at = Time.current + (decision.wait || 0)
-
-      source.with_lock do
-        source.reload
-        source.update!(
-          fetch_retry_attempt: decision.next_attempt,
-          fetch_circuit_opened_at: nil,
-          fetch_circuit_until: nil,
-          next_fetch_at: retry_at,
-          backoff_until: retry_at,
-          fetch_status: "queued"
-        )
-      end
-
-      retry_job wait: decision.wait || 0
-    end
-
-    def open_circuit!(source, decision)
-      source.with_lock do
-        source.reload
-        source.update!(
-          fetch_retry_attempt: 0,
-          fetch_circuit_opened_at: Time.current,
-          fetch_circuit_until: decision.circuit_until,
-          next_fetch_at: decision.circuit_until,
-          backoff_until: decision.circuit_until,
-          fetch_status: "failed"
-        )
-      end
-    end
-
-    def reset_retry_state!(source)
-      source.with_lock do
-        source.reload
-        source.update!(
-          fetch_retry_attempt: 0,
-          fetch_circuit_opened_at: nil,
-          fetch_circuit_until: nil
-        )
-      end
-    end
-
     def transient_error?(error)
       error.is_a?(SourceMonitor::Fetching::FetchError)
     end

data/app/jobs/source_monitor/import_opml_job.rb

@@ -1,10 +1,5 @@
 # frozen_string_literal: true
 
-require "set"
-require "source_monitor/import_sessions/entry_normalizer"
-require "source_monitor/realtime/broadcaster"
-require "source_monitor/sources/params"
-
 module SourceMonitor
   class ImportOpmlJob < ApplicationJob
     source_monitor_queue :maintenance
@@ -12,148 +7,14 @@ module SourceMonitor
     discard_on ActiveJob::DeserializationError
 
     def perform(import_session_id, import_history_id)
-      @import_session = SourceMonitor::ImportSession.find_by(id: import_session_id)
-      @import_history = SourceMonitor::ImportHistory.find_by(id: import_history_id)
+      import_session = SourceMonitor::ImportSession.find_by(id: import_session_id)
+      import_history = SourceMonitor::ImportHistory.find_by(id: import_history_id)
       return unless import_session && import_history
 
-      import_history.update_columns(started_at: Time.current) unless import_history.started_at
-
-      processed = Set.new
-
-      selected_entries.each do |entry|
-        process_entry(entry, processed)
-      end
-
-      import_history.update!(
-        imported_sources: imported_sources,
-        failed_sources: failed_sources,
-        skipped_duplicates: skipped_duplicates,
-        bulk_settings: import_session.bulk_settings.presence || {},
-        completed_at: Time.current
-      )
-
-      broadcast_completion(import_history)
-    end
-
-    private
-
-    attr_reader :import_session, :import_history
-
-    def selected_entries
-      ids = Array(import_session.selected_source_ids).map(&:to_s)
-
-      Array(import_session.parsed_sources)
-        .map { |entry| SourceMonitor::ImportSessions::EntryNormalizer.normalize(entry) }
-        .select { |entry| ids.include?(entry[:id]) }
-    end
-
-    def process_entry(entry, processed)
-      feed_url = entry[:feed_url].to_s
-
-      if feed_url.blank?
-        failed_sources << failure_payload(feed_url, "MissingFeedURL", "Feed URL is missing")
-        return
-      end
-
-      normalized_url = feed_url.downcase
-
-      if processed.include?(normalized_url)
-        skipped_duplicates << skipped_payload(feed_url, "duplicate in import selection")
-        return
-      end
-
-      if duplicate_source?(normalized_url)
-        skipped_duplicates << skipped_payload(feed_url, "already exists")
-        processed << normalized_url
-        return
-      end
-
-      source = SourceMonitor::Source.new(build_attributes(entry))
-
-      if source.save
-        imported_sources << { id: source.id, feed_url: source.feed_url, name: source.name }
-        SourceMonitor::FaviconFetchJob.perform_later(source.id) if should_fetch_favicon?(source)
-        processed << normalized_url
-      else
-        failed_sources << failure_payload(feed_url, "ValidationFailed", source.errors.full_messages.to_sentence)
-      end
-    rescue ActiveRecord::RecordNotUnique
-      skipped_duplicates << skipped_payload(feed_url, "already exists")
-      processed << normalized_url
-    rescue StandardError => error
-      failed_sources << failure_payload(feed_url, error.class.name, error.message)
-    end
-
-    def duplicate_source?(normalized_feed_url)
-      SourceMonitor::Source.where("LOWER(feed_url) = ?", normalized_feed_url).exists?
-    end
-
-    def build_attributes(entry)
-      defaults = SourceMonitor::Sources::Params.default_attributes.deep_dup
-      settings = SourceMonitor::Security::ParameterSanitizer.sanitize(import_session.bulk_settings.presence || {})
-      settings = settings.deep_symbolize_keys
-
-      defaults.merge(settings).merge(identity_attributes(entry))
-    end
-
-    def identity_attributes(entry)
-      {
-        name: entry[:title].presence || entry[:feed_url],
-        feed_url: entry[:feed_url],
-        website_url: entry[:website_url]
-      }
-    end
-
-    def imported_sources
-      @imported_sources ||= []
-    end
-
-    def failed_sources
-      @failed_sources ||= []
-    end
-
-    def skipped_duplicates
-      @skipped_duplicates ||= []
-    end
-
-    def failure_payload(feed_url, error_class, message)
-      {
-        feed_url: feed_url,
-        error_class: error_class,
-        error_message: message
-      }
-    end
-
-    def skipped_payload(feed_url, reason)
-      {
-        feed_url: feed_url,
-        reason: reason
-      }
-    end
-
-    def should_fetch_favicon?(source)
-      defined?(ActiveStorage) &&
-        SourceMonitor.config.favicons.enabled? &&
-        source.website_url.present?
-    rescue StandardError
-      false
-    end
-
-    def broadcast_completion(history)
-      return unless defined?(Turbo::StreamsChannel)
-
-      histories = SourceMonitor::ImportHistory.recent_for(history.user_id).limit(5)
-
-      Turbo::StreamsChannel.broadcast_replace_to(
-        SourceMonitor::Realtime::Broadcaster::SOURCE_INDEX_STREAM,
-        target: "source_monitor_import_history_panel",
-        html: SourceMonitor::SourcesController.render(
-          partial: "source_monitor/sources/import_history_panel",
-          locals: { import_histories: histories }
-        )
-      )
-    rescue StandardError => error
-      Rails.logger.error("[SourceMonitor::ImportOpmlJob] broadcast failed: #{error.class}: #{error.message}") if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+      SourceMonitor::ImportSessions::OPMLImporter.new(
+        import_session: import_session,
+        import_history: import_history
+      ).call
     end
   end
 end

data/app/jobs/source_monitor/import_session_health_check_job.rb

@@ -4,90 +4,29 @@ module SourceMonitor
   class ImportSessionHealthCheckJob < ApplicationJob
     source_monitor_queue :maintenance
 
-    require "source_monitor/health/import_source_health_check"
-    require "source_monitor/import_sessions/entry_normalizer"
-    require "source_monitor/import_sessions/health_check_broadcaster"
-
     discard_on ActiveJob::DeserializationError
 
+    rescue_from ActiveRecord::Deadlocked do |error|
+      Rails.logger&.warn("[SourceMonitor::ImportSessionHealthCheckJob] Deadlock: #{error.message}")
+      retry_job(wait: 2.seconds + rand(3).seconds)
+    end
+
     def perform(import_session_id, entry_id)
       import_session = SourceMonitor::ImportSession.find_by(id: import_session_id)
       return unless import_session
-      return unless active_for?(import_session)
-
-      result = perform_health_check(import_session, entry_id)
-      return unless result
-
-      updated_entry = nil
-
-      import_session.with_lock do
-        import_session.reload
-        return unless active_for?(import_session)
-
-        entries = Array(import_session.parsed_sources).map(&:to_h)
-        index = entries.index { |candidate| entry_id_for(candidate) == entry_id.to_s }
-        return unless index
 
-        entries[index] = entries[index].merge(
-          "health_status" => result.status,
-          "health_error" => result.error_message
+      SourceMonitor::ImportSessions::HealthCheckUpdater.new(
+        import_session: import_session,
+        entry_id: entry_id
+      ).call
+    rescue ActiveRecord::Deadlocked
+      raise # re-raise so rescue_from handler catches it
+    rescue StandardError => error
+      if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+        Rails.logger.error(
+          "[SourceMonitor::ImportSessionHealthCheckJob] #{error.class}: #{error.message}"
         )
-
-        selected_ids = Array(import_session.selected_source_ids).map(&:to_s)
-        selected_ids -= [ entry_id.to_s ] if result.status == "unhealthy"
-
-        attrs = {
-          parsed_sources: entries,
-          selected_source_ids: selected_ids,
-          health_check_completed_at: completion_time(entries, import_session.health_check_targets)
-        }.compact
-
-        import_session.update!(attrs)
-        normalized_entry = SourceMonitor::ImportSessions::EntryNormalizer.normalize(entries[index])
-        updated_entry = normalized_entry.merge(selected: selected_ids.include?(entry_id.to_s))
       end
-
-      broadcaster = SourceMonitor::ImportSessions::HealthCheckBroadcaster.new(import_session)
-      broadcaster.broadcast_row(updated_entry) if updated_entry
-      broadcaster.broadcast_progress
-    rescue StandardError => error
-      Rails.logger.error(
-        "[SourceMonitor::ImportSessionHealthCheckJob] #{error.class}: #{error.message}"
-      ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
-    end
-
-    private
-
-    def active_for?(import_session)
-      import_session.current_step == "health_check" && import_session.health_checks_active?
-    end
-
-    def perform_health_check(import_session, entry_id)
-      entry = find_entry(import_session, entry_id)
-      return unless entry
-
-      SourceMonitor::Health::ImportSourceHealthCheck.new(feed_url: entry_feed_url(entry)).call
-    end
-
-    def find_entry(import_session, entry_id)
-      Array(import_session.parsed_sources).find { |entry| entry_id_for(entry) == entry_id.to_s }
-    end
-
-    def entry_id_for(entry)
-      entry.to_h["id"].presence || entry.to_h[:id].presence || entry.to_h["feed_url"].to_s
-    end
-
-    def entry_feed_url(entry)
-      entry.to_h["feed_url"] || entry.to_h[:feed_url]
-    end
-
-    def completion_time(entries, targets)
-      normalized = Array(entries).map { |entry| SourceMonitor::ImportSessions::EntryNormalizer.normalize(entry) }
-      filtered = normalized.select { |entry| targets.include?(entry[:id]) }
-      return nil if filtered.empty?
-
-      completed = filtered.count { |entry| %w[healthy unhealthy].include?(entry[:health_status].to_s) }
-      completed >= filtered.size ? Time.current : nil
     end
   end
 end

data/app/jobs/source_monitor/item_cleanup_job.rb

@@ -6,6 +6,11 @@ module SourceMonitor
 
     source_monitor_queue :maintenance
 
+    rescue_from ActiveRecord::Deadlocked do |error|
+      Rails.logger&.warn("[SourceMonitor::ItemCleanupJob] Deadlock: #{error.message}")
+      retry_job(wait: 2.seconds + rand(3).seconds)
+    end
+
     def perform(options = nil)
       options = SourceMonitor::Jobs::CleanupOptions.normalize(options)
 

data/app/jobs/source_monitor/log_cleanup_job.rb

@@ -7,6 +7,11 @@ module SourceMonitor
 
     source_monitor_queue :maintenance
 
+    rescue_from ActiveRecord::Deadlocked do |error|
+      Rails.logger&.warn("[SourceMonitor::LogCleanupJob] Deadlock: #{error.message}")
+      retry_job(wait: 2.seconds + rand(3).seconds)
+    end
+
     def perform(options = nil)
       options = SourceMonitor::Jobs::CleanupOptions.normalize(options)
 
@@ -36,12 +41,18 @@ module SourceMonitor
 
     def prune_fetch_logs(cutoff)
       SourceMonitor::FetchLog.where(SourceMonitor::FetchLog.arel_table[:started_at].lt(cutoff))
-        .in_batches(of: 500) { |batch| batch.delete_all }
+        .in_batches(of: 500) do |batch|
+          SourceMonitor::LogEntry.where(loggable_type: "SourceMonitor::FetchLog", loggable_id: batch.select(:id)).delete_all
+          batch.delete_all
+        end
     end
 
     def prune_scrape_logs(cutoff)
       SourceMonitor::ScrapeLog.where(SourceMonitor::ScrapeLog.arel_table[:started_at].lt(cutoff))
-        .in_batches(of: 500) { |batch| batch.delete_all }
+        .in_batches(of: 500) do |batch|
+          SourceMonitor::LogEntry.where(loggable_type: "SourceMonitor::ScrapeLog", loggable_id: batch.select(:id)).delete_all
+          batch.delete_all
+        end
     end
   end
 end

data/app/jobs/source_monitor/schedule_fetches_job.rb

@@ -4,9 +4,17 @@ module SourceMonitor
   class ScheduleFetchesJob < ApplicationJob
     source_monitor_queue :fetch
 
+    rescue_from ActiveRecord::Deadlocked do |error|
+      Rails.logger&.warn("[SourceMonitor::ScheduleFetchesJob] Deadlock: #{error.message}")
+      retry_job(wait: 2.seconds + rand(3).seconds)
+    end
+
     def perform(options = nil)
       limit = extract_limit(options)
       SourceMonitor::Scheduler.run(limit:)
+    rescue StandardError => error
+      Rails.logger&.error("[SourceMonitor::ScheduleFetchesJob] #{error.class}: #{error.message}")
+      raise
     end
 
     private

data/app/jobs/source_monitor/scrape_item_job.rb

@@ -6,62 +6,16 @@ module SourceMonitor
 
     discard_on ActiveJob::DeserializationError
 
+    rescue_from ActiveRecord::Deadlocked do |error|
+      Rails.logger&.warn("[SourceMonitor::ScrapeItemJob] Deadlock: #{error.message}")
+      retry_job(wait: 2.seconds + rand(3).seconds)
+    end
+
     def perform(item_id)
-      log("job:start", item_id: item_id)
       item = SourceMonitor::Item.includes(:source).find_by(id: item_id)
       return unless item
 
-      source = item.source
-      unless source&.scraping_enabled?
-        log("job:skipped_scraping_disabled", item: item)
-        SourceMonitor::Scraping::State.clear_inflight!(item)
-        return
-      end
-
-      remaining = time_until_scrape_allowed(source)
-      if remaining&.positive?
-        SourceMonitor::Scraping::State.clear_inflight!(item)
-        self.class.set(wait: remaining.seconds).perform_later(item_id)
-        log("job:deferred", item: item, wait_seconds: remaining)
-        return
-      end
-
-      SourceMonitor::Scraping::State.mark_processing!(item)
-      SourceMonitor::Scraping::ItemScraper.new(item:, source:).call
-      log("job:completed", item: item, status: item.scrape_status)
-    rescue StandardError => error
-      log("job:error", item: item, error: error.message)
-      SourceMonitor::Scraping::State.mark_failed!(item)
-      raise
-    ensure
-      SourceMonitor::Scraping::State.clear_inflight!(item) if item
-    end
-
-    private
-
-    def time_until_scrape_allowed(source)
-      interval = source.min_scrape_interval || SourceMonitor.config.scraping.min_scrape_interval
-      return nil if interval.nil? || interval <= 0
-
-      last_scrape_at = source.scrape_logs.maximum(:started_at)
-      return nil unless last_scrape_at
-
-      elapsed = Time.current - last_scrape_at
-      remaining = interval - elapsed
-      remaining.positive? ? remaining.ceil : nil
-    end
-
-    def log(stage, item: nil, item_id: nil, **extra)
-      return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
-
-      payload = {
-        stage: "SourceMonitor::ScrapeItemJob##{stage}",
-        item_id: item&.id || item_id,
-        source_id: item&.source_id
-      }.merge(extra.compact)
-      Rails.logger.info("[SourceMonitor::ManualScrape] #{payload.to_json}")
-    rescue StandardError
-      nil
+      SourceMonitor::Scraping::Runner.new(item).call
     end
   end
 end

data/app/jobs/source_monitor/source_health_check_job.rb

@@ -10,78 +10,7 @@ module SourceMonitor
       source = SourceMonitor::Source.find_by(id: source_id)
       return unless source
 
-      result = SourceMonitor::Health::SourceHealthCheck.new(source: source).call
-      broadcast_outcome(source, result)
-      trigger_fetch_if_degraded(source, result)
-      result
-    rescue StandardError => error
-      Rails.logger&.error(
-        "[SourceMonitor::SourceHealthCheckJob] error for source #{source_id}: #{error.class}: #{error.message}"
-      ) if defined?(Rails) && Rails.respond_to?(:logger)
-
-      record_unexpected_failure(source, error) if source
-      broadcast_outcome(source, nil, error) if source
-      nil
-    end
-
-    DEGRADED_STATUSES = %w[declining failing].freeze
-
-    private
-
-    def trigger_fetch_if_degraded(source, result)
-      return unless result&.success?
-      return unless DEGRADED_STATUSES.include?(source.health_status.to_s)
-
-      SourceMonitor::FetchFeedJob.perform_later(source.id, force: true)
-    end
-
-    def record_unexpected_failure(source, error)
-      SourceMonitor::HealthCheckLog.create!(
-        source: source,
-        success: false,
-        started_at: Time.current,
-        completed_at: Time.current,
-        duration_ms: 0,
-        error_class: error.class.name,
-        error_message: error.message
-      )
-    rescue StandardError
-      nil
-    end
-
-    def broadcast_outcome(source, result, error = nil)
-      SourceMonitor::Realtime.broadcast_source(source)
-
-      message, level = toast_payload(source, result, error)
-      return if message.blank?
-
-      SourceMonitor::Realtime.broadcast_toast(message:, level:)
-    end
-
-    def toast_payload(source, result, error)
-      if error
-        return [
-          "Health check failed for #{source.name}: #{error.message}",
-          :error
-        ]
-      end
-
-      if result&.success?
-        [
-          "Health check succeeded for #{source.name}.",
-          :success
-        ]
-      else
-        failure_reason = result&.error&.message
-        http_status = result&.log&.http_status
-        message = "Health check failed for #{source.name}"
-        message += " (HTTP #{http_status})" if http_status.present?
-        message += ": #{failure_reason}" if failure_reason.present?
-        [
-          "#{message}.",
-          :error
-        ]
-      end
+      SourceMonitor::Health::SourceHealthCheckOrchestrator.new(source).call
     end
   end
 end