source_monitor 0.1.1

data/app/jobs/source_monitor/fetch_feed_job.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class FetchFeedJob < ApplicationJob
+    FETCH_CONCURRENCY_RETRY_WAIT = 30.seconds
+    EARLY_EXECUTION_LEEWAY = 30.seconds
+
+    source_monitor_queue :fetch
+
+    discard_on ActiveJob::DeserializationError
+    retry_on SourceMonitor::Fetching::FetchRunner::ConcurrencyError,
+             wait: FETCH_CONCURRENCY_RETRY_WAIT,
+             attempts: 5
+
+    def perform(source_id, force: false)
+      source = SourceMonitor::Source.find_by(id: source_id)
+      return unless source
+
+      return unless should_run?(source, force: force)
+
+      SourceMonitor::Fetching::FetchRunner.new(source: source, force: force).run
+    rescue SourceMonitor::Fetching::FetchError => error
+      handle_transient_error(source, error)
+    end
+
+    private
+
+    def should_run?(source, force:)
+      return true if force
+
+      status = source.fetch_status.to_s
+      return true if %w[queued fetching].include?(status)
+
+      next_fetch_at = source.next_fetch_at
+      return true if next_fetch_at.blank?
+
+      next_fetch_at <= Time.current + EARLY_EXECUTION_LEEWAY
+    end
+
+    def handle_transient_error(source, error)
+      raise error unless transient_error?(error) && source
+
+      decision = SourceMonitor::Fetching::RetryPolicy.new(source:, error:, now: Time.current).decision
+      return raise error unless decision
+
+      if decision.retry?
+        enqueue_retry!(source, decision)
+      elsif decision.open_circuit?
+        open_circuit!(source, decision)
+        raise error
+      else
+        reset_retry_state!(source)
+        raise error
+      end
+    rescue StandardError => policy_error
+      log_retry_failure(source, error, policy_error)
+      raise error
+    end
+
+    def enqueue_retry!(source, decision)
+      retry_at = Time.current + (decision.wait || 0)
+
+      source.with_lock do
+        source.reload
+        source.update!(
+          fetch_retry_attempt: decision.next_attempt,
+          fetch_circuit_opened_at: nil,
+          fetch_circuit_until: nil,
+          next_fetch_at: retry_at,
+          backoff_until: retry_at,
+          fetch_status: "queued"
+        )
+      end
+
+      retry_job wait: decision.wait || 0
+    end
+
+    def open_circuit!(source, decision)
+      source.with_lock do
+        source.reload
+        source.update!(
+          fetch_retry_attempt: 0,
+          fetch_circuit_opened_at: Time.current,
+          fetch_circuit_until: decision.circuit_until,
+          next_fetch_at: decision.circuit_until,
+          backoff_until: decision.circuit_until,
+          fetch_status: "failed"
+        )
+      end
+    end
+
+    def reset_retry_state!(source)
+      source.with_lock do
+        source.reload
+        source.update!(
+          fetch_retry_attempt: 0,
+          fetch_circuit_opened_at: nil,
+          fetch_circuit_until: nil
+        )
+      end
+    end
+
+    def transient_error?(error)
+      error.is_a?(SourceMonitor::Fetching::FetchError)
+    end
+
+    def log_retry_failure(source, original_error, policy_error)
+      return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+      message = "[SourceMonitor::FetchFeedJob] Failed to schedule retry for source #{source&.id}: " \
+                "#{original_error.class}: #{original_error.message} (policy error: #{policy_error.class})"
+      Rails.logger.error(message)
+    rescue StandardError
+      nil
+    end
+  end
+end

data/app/jobs/source_monitor/item_cleanup_job.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class ItemCleanupJob < ApplicationJob
+    DEFAULT_BATCH_SIZE = 100
+
+    source_monitor_queue :fetch
+
+    def perform(options = nil)
+      options = SourceMonitor::Jobs::CleanupOptions.normalize(options)
+
+      scope = resolve_scope(options)
+      batch_size = SourceMonitor::Jobs::CleanupOptions.batch_size(options, default: DEFAULT_BATCH_SIZE)
+      now = SourceMonitor::Jobs::CleanupOptions.resolve_time(options[:now])
+      strategy = resolve_strategy(options)
+      pruner_class = options[:retention_pruner_class] || SourceMonitor::Items::RetentionPruner
+
+      scope.find_in_batches(batch_size:) do |batch|
+        batch.each do |source|
+          pruner_class.call(source:, now:, strategy:)
+        end
+      end
+    end
+
+    private
+
+    def resolve_scope(options)
+      relation = options[:source_scope] || SourceMonitor::Source.all
+      ids = SourceMonitor::Jobs::CleanupOptions.extract_ids([ options[:source_ids], options[:source_id] ])
+
+      if ids.any?
+        relation.where(id: ids)
+      else
+        relation
+      end
+    end
+
+    def resolve_strategy(options)
+      if options.key?(:strategy)
+        options[:strategy]
+      elsif options.key?(:soft_delete)
+        ActiveModel::Type::Boolean.new.cast(options[:soft_delete]) ? :soft_delete : :destroy
+      else
+        SourceMonitor.config.retention.strategy
+      end
+    end
+  end
+end

data/app/jobs/source_monitor/log_cleanup_job.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class LogCleanupJob < ApplicationJob
+    DEFAULT_FETCH_LOG_RETENTION_DAYS = 90
+    DEFAULT_SCRAPE_LOG_RETENTION_DAYS = 45
+
+    source_monitor_queue :fetch
+
+    def perform(options = nil)
+      options = SourceMonitor::Jobs::CleanupOptions.normalize(options)
+
+      now = SourceMonitor::Jobs::CleanupOptions.resolve_time(options[:now])
+      fetch_cutoff = resolve_cutoff(now:, days: options[:fetch_logs_older_than_days], default: DEFAULT_FETCH_LOG_RETENTION_DAYS)
+      scrape_cutoff = resolve_cutoff(now:, days: options[:scrape_logs_older_than_days], default: DEFAULT_SCRAPE_LOG_RETENTION_DAYS)
+
+      prune_fetch_logs(fetch_cutoff) if fetch_cutoff
+      prune_scrape_logs(scrape_cutoff) if scrape_cutoff
+    end
+
+    private
+
+    def resolve_cutoff(now:, days:, default:)
+      resolved_days =
+        if days.nil?
+          default
+        else
+          SourceMonitor::Jobs::CleanupOptions.integer(days)
+        end
+
+      return nil unless resolved_days
+      return nil if resolved_days <= 0
+
+      now - resolved_days.days
+    end
+
+    def prune_fetch_logs(cutoff)
+      SourceMonitor::FetchLog.where(SourceMonitor::FetchLog.arel_table[:started_at].lt(cutoff))
+        .in_batches(of: 500) { |batch| batch.delete_all }
+    end
+
+    def prune_scrape_logs(cutoff)
+      SourceMonitor::ScrapeLog.where(SourceMonitor::ScrapeLog.arel_table[:started_at].lt(cutoff))
+        .in_batches(of: 500) { |batch| batch.delete_all }
+    end
+  end
+end

data/app/jobs/source_monitor/schedule_fetches_job.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class ScheduleFetchesJob < ApplicationJob
+    source_monitor_queue :fetch
+
+    def perform(options = nil)
+      limit = extract_limit(options)
+      SourceMonitor::Scheduler.run(limit:)
+    end
+
+    private
+
+    def extract_limit(options)
+      options_hash =
+        case options
+        when nil then {}
+        when Hash then options
+        else {}
+        end
+
+      if options_hash.respond_to?(:symbolize_keys)
+        options_hash = options_hash.symbolize_keys
+      end
+
+      options_hash[:limit] || SourceMonitor::Scheduler::DEFAULT_BATCH_SIZE
+    end
+  end
+end

data/app/jobs/source_monitor/scrape_item_job.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class ScrapeItemJob < ApplicationJob
+    source_monitor_queue :scrape
+
+    discard_on ActiveJob::DeserializationError
+
+    def perform(item_id)
+      log("job:start", item_id: item_id)
+      item = SourceMonitor::Item.includes(:source).find_by(id: item_id)
+      return unless item
+
+      source = item.source
+      unless source&.scraping_enabled?
+        log("job:skipped_scraping_disabled", item: item)
+        SourceMonitor::Scraping::State.clear_inflight!(item)
+        return
+      end
+
+      SourceMonitor::Scraping::State.mark_processing!(item)
+      SourceMonitor::Scraping::ItemScraper.new(item:, source:).call
+      log("job:completed", item: item, status: item.scrape_status)
+    rescue StandardError => error
+      log("job:error", item: item, error: error.message)
+      SourceMonitor::Scraping::State.mark_failed!(item)
+      raise
+    ensure
+      SourceMonitor::Scraping::State.clear_inflight!(item) if item
+    end
+
+    private
+
+    def log(stage, item: nil, item_id: nil, **extra)
+      return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+      payload = {
+        stage: "SourceMonitor::ScrapeItemJob##{stage}",
+        item_id: item&.id || item_id,
+        source_id: item&.source_id
+      }.merge(extra.compact)
+      Rails.logger.info("[SourceMonitor::ManualScrape] #{payload.to_json}")
+    rescue StandardError
+      nil
+    end
+  end
+end

data/app/jobs/source_monitor/source_health_check_job.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class SourceHealthCheckJob < ApplicationJob
+    source_monitor_queue :fetch
+
+    discard_on ActiveJob::DeserializationError
+
+    def perform(source_id)
+      source = SourceMonitor::Source.find_by(id: source_id)
+      return unless source
+
+      result = SourceMonitor::Health::SourceHealthCheck.new(source: source).call
+      broadcast_outcome(source, result)
+      result
+    rescue StandardError => error
+      Rails.logger&.error(
+        "[SourceMonitor::SourceHealthCheckJob] error for source #{source_id}: #{error.class}: #{error.message}"
+      ) if defined?(Rails) && Rails.respond_to?(:logger)
+
+      record_unexpected_failure(source, error) if source
+      broadcast_outcome(source, nil, error) if source
+      nil
+    end
+
+    private
+
+    def record_unexpected_failure(source, error)
+      SourceMonitor::HealthCheckLog.create!(
+        source: source,
+        success: false,
+        started_at: Time.current,
+        completed_at: Time.current,
+        duration_ms: 0,
+        error_class: error.class.name,
+        error_message: error.message
+      )
+    rescue StandardError
+      nil
+    end
+
+    def broadcast_outcome(source, result, error = nil)
+      SourceMonitor::Realtime.broadcast_source(source)
+
+      message, level = toast_payload(source, result, error)
+      return if message.blank?
+
+      SourceMonitor::Realtime.broadcast_toast(message:, level:)
+    end
+
+    def toast_payload(source, result, error)
+      if error
+        return [
+          "Health check failed for #{source.name}: #{error.message}",
+          :error
+        ]
+      end
+
+      if result&.success?
+        [
+          "Health check succeeded for #{source.name}.",
+          :success
+        ]
+      else
+        failure_reason = result&.error&.message
+        http_status = result&.log&.http_status
+        message = "Health check failed for #{source.name}"
+        message += " (HTTP #{http_status})" if http_status.present?
+        message += ": #{failure_reason}" if failure_reason.present?
+        [
+          "#{message}.",
+          :error
+        ]
+      end
+    end
+  end
+end

data/app/mailers/source_monitor/application_mailer.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  if defined?(::ActionMailer::Base)
+    class ApplicationMailer < ::ActionMailer::Base
+      default from: "from@example.com"
+      layout "mailer"
+    end
+  else
+    # :nocov:
+    # Define a no-op mailer so API-only host apps (which skip Action Mailer)
+    # can autoload this constant without pulling in the framework.
+    class ApplicationMailer
+    end
+    # :nocov:
+  end
+end

data/app/models/concerns/source_monitor/loggable.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Loggable
+    extend ActiveSupport::Concern
+
+    included do
+      attribute :metadata, default: -> { {} }
+
+      validates :started_at, presence: true
+      validates :duration_ms, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
+
+      scope :recent, -> { order(started_at: :desc) }
+      scope :successful, -> { where(success: true) }
+      scope :failed, -> { where(success: false) }
+    end
+  end
+end

data/app/models/source_monitor/application_record.rb

@@ -0,0 +1,5 @@
+module SourceMonitor
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/source_monitor/fetch_log.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class FetchLog < ApplicationRecord
+    include SourceMonitor::Loggable
+
+    belongs_to :source, class_name: "SourceMonitor::Source", inverse_of: :fetch_logs
+    has_one :log_entry, as: :loggable, class_name: "SourceMonitor::LogEntry", inverse_of: :loggable, dependent: :destroy
+
+    attribute :items_created, :integer, default: 0
+    attribute :items_updated, :integer, default: 0
+    attribute :items_failed, :integer, default: 0
+    attribute :http_response_headers, default: -> { {} }
+
+    validates :source, presence: true
+    validates :items_created, :items_updated, :items_failed,
+              numericality: { greater_than_or_equal_to: 0 }
+
+    scope :for_job, ->(job_id) { where(job_id:) }
+
+    SourceMonitor::ModelExtensions.register(self, :fetch_log)
+
+    after_save :sync_log_entry
+
+    private
+
+    def sync_log_entry
+      SourceMonitor::Logs::EntrySync.call(self)
+    end
+  end
+end

data/app/models/source_monitor/health_check_log.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class HealthCheckLog < ApplicationRecord
+    include SourceMonitor::Loggable
+
+    belongs_to :source, class_name: "SourceMonitor::Source", inverse_of: :health_check_logs
+    has_one :log_entry,
+            as: :loggable,
+            class_name: "SourceMonitor::LogEntry",
+            inverse_of: :loggable,
+            dependent: :destroy
+
+    attribute :http_response_headers, default: -> { {} }
+
+    validates :source, presence: true
+
+    SourceMonitor::ModelExtensions.register(self, :health_check_log)
+
+    after_save :sync_log_entry
+
+    private
+
+    def sync_log_entry
+      SourceMonitor::Logs::EntrySync.call(self)
+    end
+  end
+end

data/app/models/source_monitor/item.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "source_monitor/models/url_normalizable"
+
+module SourceMonitor
+  class Item < ApplicationRecord
+    include SourceMonitor::Models::UrlNormalizable
+
+    belongs_to :source, class_name: "SourceMonitor::Source", inverse_of: :items, counter_cache: true
+    has_one :item_content, class_name: "SourceMonitor::ItemContent", inverse_of: :item, dependent: :destroy, autosave: true
+    has_many :scrape_logs, class_name: "SourceMonitor::ScrapeLog", inverse_of: :item, dependent: :destroy
+    has_many :log_entries, class_name: "SourceMonitor::LogEntry", inverse_of: :item, dependent: :destroy
+
+    # Explicit scope for active (non-deleted) items - no default_scope to avoid anti-pattern
+    scope :active, -> { where(deleted_at: nil) }
+    scope :with_deleted, -> { unscope(where: :deleted_at) }
+    scope :only_deleted, -> { where.not(deleted_at: nil) }
+
+    normalizes_urls :url, :canonical_url, :comments_url
+    validates_url_format :url, :canonical_url, :comments_url
+
+    validates :source, presence: true
+    validates :guid, presence: true, uniqueness: { scope: :source_id, case_sensitive: false }
+    validates :content_fingerprint, uniqueness: { scope: :source_id }, allow_blank: true
+    validates :url, presence: true
+
+    scope :recent, -> { active.order(Arel.sql("published_at DESC NULLS LAST, created_at DESC")) }
+    scope :published, -> { active.where.not(published_at: nil) }
+    scope :pending_scrape, -> { active.where(scraped_at: nil) }
+    scope :failed_scrape, -> { active.where(scrape_status: "failed") }
+
+    delegate :scraped_html, :scraped_content, to: :item_content, allow_nil: true
+
+    SourceMonitor::ModelExtensions.register(self, :item)
+
+    class << self
+      def ransackable_attributes(_auth_object = nil)
+        %w[title summary url published_at created_at scrape_status]
+      end
+
+      def ransackable_associations(_auth_object = nil)
+        %w[source]
+      end
+    end
+
+    def scraped_html=(value)
+      assign_content_attribute(:scraped_html, value)
+    end
+
+    def scraped_content=(value)
+      assign_content_attribute(:scraped_content, value)
+    end
+
+    def deleted?
+      deleted_at.present?
+    end
+
+    def soft_delete!(timestamp: Time.current)
+      return if deleted?
+
+      self.class.transaction do
+        timestamp = timestamp.in_time_zone if timestamp.respond_to?(:in_time_zone)
+        timestamp ||= Time.current
+
+        update_columns(
+          deleted_at: timestamp,
+          updated_at: timestamp
+        )
+
+        SourceMonitor::Source.decrement_counter(:items_count, source_id) if source_id
+      end
+    end
+
+    private
+
+    # Item content lives in a separate row that we only create when rich content exists.
+    # This helper keeps the association lazy-loaded, ensures updates route through the
+    # join model, and tears it back down once both scraped fields are blank again.
+    def assign_content_attribute(attribute, value)
+      ensure_item_content_presence(value)
+      item_content&.public_send("#{attribute}=", value)
+      cleanup_item_content_if_blank
+    end
+
+    def ensure_item_content_presence(value)
+      return if item_content.present? || value.nil?
+
+      build_item_content
+    end
+
+    def cleanup_item_content_if_blank
+      return unless item_content
+      return if item_content.scraped_html.present? || item_content.scraped_content.present?
+
+      if item_content.persisted?
+        item_content.mark_for_destruction
+      else
+        association(:item_content).reset
+      end
+    end
+  end
+end

data/app/models/source_monitor/item_content.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class ItemContent < ApplicationRecord
+    belongs_to :item, class_name: "SourceMonitor::Item", inverse_of: :item_content, touch: true
+
+    validates :item, presence: true
+
+    SourceMonitor::ModelExtensions.register(self, :item_content)
+  end
+end

data/app/models/source_monitor/log_entry.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class LogEntry < ApplicationRecord
+    self.table_name = "sourcemon_log_entries"
+
+    delegated_type :loggable, types: %w[SourceMonitor::FetchLog SourceMonitor::ScrapeLog SourceMonitor::HealthCheckLog]
+
+    belongs_to :source, class_name: "SourceMonitor::Source", inverse_of: :log_entries
+    belongs_to :item, class_name: "SourceMonitor::Item", inverse_of: :log_entries, optional: true
+
+    validates :started_at, presence: true
+    validates :source, presence: true
+
+    scope :recent, -> { order(started_at: :desc) }
+
+    SourceMonitor::ModelExtensions.register(self, :log_entry)
+
+    class << self
+      def ransackable_attributes(_auth_object = nil)
+        %w[
+          success
+          started_at
+          http_status
+          scraper_adapter
+          error_message
+          error_class
+          loggable_type
+        ]
+      end
+
+      def ransackable_associations(_auth_object = nil)
+        %w[source item loggable]
+      end
+    end
+
+    def fetch?
+      loggable_type == FetchLog.sti_name
+    end
+
+    def scrape?
+      loggable_type == ScrapeLog.sti_name
+    end
+
+    def health_check?
+      loggable_type == HealthCheckLog.sti_name
+    end
+
+    def log_type
+      return :fetch if fetch?
+      return :scrape if scrape?
+
+      :health_check
+    end
+  end
+end

data/app/models/source_monitor/scrape_log.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class ScrapeLog < ApplicationRecord
+    include SourceMonitor::Loggable
+
+    belongs_to :item, class_name: "SourceMonitor::Item", inverse_of: :scrape_logs
+    belongs_to :source, class_name: "SourceMonitor::Source", inverse_of: :scrape_logs
+    has_one :log_entry, as: :loggable, class_name: "SourceMonitor::LogEntry", inverse_of: :loggable, dependent: :destroy
+
+    validates :item, :source, presence: true
+    validates :content_length, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
+    validate :source_matches_item
+
+    SourceMonitor::ModelExtensions.register(self, :scrape_log)
+
+    after_save :sync_log_entry
+
+    private
+
+    def source_matches_item
+      return if item.nil? || source.nil?
+
+      errors.add(:source, "must match item source") if item.source_id != source_id
+    end
+
+    def sync_log_entry
+      SourceMonitor::Logs::EntrySync.call(self)
+    end
+  end
+end