source_monitor 0.1.1

data/lib/source_monitor/fetching/fetch_runner.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "source_monitor/fetching/advisory_lock"
+require "source_monitor/fetching/completion/retention_handler"
+require "source_monitor/fetching/completion/follow_up_handler"
+require "source_monitor/fetching/completion/event_publisher"
+
+module SourceMonitor
+  module Fetching
+    # Coordinates execution of FeedFetcher while ensuring we do not run more than
+    # one fetch per-source concurrently. The runner also centralizes the logic
+    # for queuing follow-up scraping jobs so both background jobs and manual UI
+    # entry points share the same behavior.
+    class FetchRunner
+      LOCK_NAMESPACE = 1_746_219
+
+      class ConcurrencyError < StandardError; end
+
+      attr_reader :source, :fetcher_class, :force, :lock, :retention_handler, :follow_up_handler, :event_publisher
+
+      def initialize(source:, fetcher_class: SourceMonitor::Fetching::FeedFetcher, scrape_job_class: SourceMonitor::ScrapeItemJob, scrape_enqueuer_class: SourceMonitor::Scraping::Enqueuer, retention_pruner_class: SourceMonitor::Items::RetentionPruner, lock_factory: SourceMonitor::Fetching::AdvisoryLock, retention_handler: nil, follow_up_handler: nil, event_publisher: nil, force: false)
+        @source = source
+        @fetcher_class = fetcher_class
+        @force = force
+        @lock = lock_factory.new(
+          namespace: LOCK_NAMESPACE,
+          key: source.id,
+          connection_pool: ActiveRecord::Base.connection_pool
+        )
+        @retention_handler = retention_handler || SourceMonitor::Fetching::Completion::RetentionHandler.new(pruner: retention_pruner_class)
+        @follow_up_handler = follow_up_handler || SourceMonitor::Fetching::Completion::FollowUpHandler.new(enqueuer_class: scrape_enqueuer_class, job_class: scrape_job_class)
+        @event_publisher = event_publisher || SourceMonitor::Fetching::Completion::EventPublisher.new
+        @retry_scheduled = false
+      end
+
+      def self.run(source:, **options)
+        new(source:, **options).run
+      end
+
+      def self.enqueue(source_or_id, force: false)
+        source = resolve_source(source_or_id)
+        return unless source
+
+        # Don't broadcast here - controller handles immediate UI update
+        source.update!(fetch_status: "queued")
+        SourceMonitor::FetchFeedJob.perform_later(source.id, force: force)
+      end
+
+      def run
+        return skip_due_to_circuit if circuit_blocked?
+
+        @retry_scheduled = false
+        result = nil
+
+        lock.with_lock do
+          mark_fetching!
+          result = fetcher_class.new(source: source).call
+          retention_handler.call(source:, result:)
+          follow_up_handler.call(source:, result:)
+          schedule_retry_if_needed(result)
+          mark_complete!(result)
+        end
+
+        event_publisher.call(source:, result:)
+        result
+      rescue SourceMonitor::Fetching::AdvisoryLock::NotAcquiredError => error
+        raise ConcurrencyError, error.message
+      rescue StandardError => error
+        mark_failed!(error)
+        event_publisher.call(source:, result: nil)
+        raise
+      end
+
+      private
+
+      def self.resolve_source(source_or_id)
+        return source_or_id if source_or_id.is_a?(SourceMonitor::Source)
+
+        SourceMonitor::Source.find_by(id: source_or_id)
+      end
+      private_class_method :resolve_source
+
+      def self.update_source_state!(source, attrs)
+        source.update!(attrs)
+        SourceMonitor::Realtime.broadcast_source(source)
+      rescue StandardError => error
+        Rails.logger.error(
+          "[SourceMonitor] Failed to update fetch state for source #{source.id}: #{error.class}: #{error.message}"
+        ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+      end
+      private_class_method :update_source_state!
+
+      def circuit_blocked?
+        !force && source.fetch_circuit_open?
+      end
+
+      def skip_due_to_circuit
+        update_source_state(fetch_status: "failed")
+        event_publisher.call(source:, result: nil)
+        nil
+      end
+
+      def mark_fetching!
+        update_source_state(fetch_status: "fetching", last_fetch_started_at: Time.current)
+      end
+
+      def mark_complete!(result)
+        status = result&.status
+        new_status =
+          if @retry_scheduled
+            "queued"
+          else
+            status == :failed ? "failed" : "idle"
+          end
+        update_source_state(fetch_status: new_status)
+      end
+
+      def mark_failed!(_error)
+        @retry_scheduled = false
+        update_source_state(fetch_status: "failed")
+      end
+
+      def update_source_state(attrs)
+        self.class.send(:update_source_state!, source, attrs)
+      end
+
+      def schedule_retry_if_needed(result)
+        decision = result&.retry_decision
+        return unless decision&.retry?
+
+        wait = decision.wait || 0
+        queue = SourceMonitor::FetchFeedJob.set(wait: wait)
+        queue.perform_later(source.id, force: false)
+        @retry_scheduled = true
+      rescue StandardError => error
+        Rails.logger.error(
+          "[SourceMonitor] Failed to enqueue retry for source #{source.id}: #{error.class}: #{error.message}"
+        ) if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+      end
+    end
+  end
+end

data/lib/source_monitor/fetching/retry_policy.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/numeric/time"
+
+module SourceMonitor
+  module Fetching
+    class RetryPolicy
+      Decision = Struct.new(
+        :retry?,
+        :wait,
+        :next_attempt,
+        :open_circuit?,
+        :circuit_until,
+        keyword_init: true
+      )
+
+      DEFAULTS = {
+        timeout: { attempts: 2, wait: 2.minutes, circuit_wait: 1.hour },
+        connection: { attempts: 3, wait: 5.minutes, circuit_wait: 1.hour },
+        http_429: { attempts: 2, wait: 15.minutes, circuit_wait: 90.minutes },
+        http_5xx: { attempts: 2, wait: 10.minutes, circuit_wait: 90.minutes },
+        http_4xx: { attempts: 1, wait: 45.minutes, circuit_wait: 2.hours },
+        parsing: { attempts: 1, wait: 30.minutes, circuit_wait: 2.hours },
+        unexpected: { attempts: 1, wait: 30.minutes, circuit_wait: 2.hours },
+        fallback: { attempts: 2, wait: 10.minutes, circuit_wait: 90.minutes }
+      }.freeze
+
+      attr_reader :source, :error, :now
+
+      def initialize(source:, error:, now: Time.current)
+        @source = source
+        @error = error
+        @now = now
+      end
+
+      def decision
+        policy = DEFAULTS[policy_key]
+        attempts = policy.fetch(:attempts)
+        wait_duration = policy.fetch(:wait)
+        circuit_duration = policy.fetch(:circuit_wait)
+
+        current_attempt = source.fetch_retry_attempt.to_i
+        next_attempt = current_attempt + 1
+
+        if next_attempt <= attempts
+          Decision.new(
+            retry?: true,
+            wait: wait_duration,
+            next_attempt: next_attempt,
+            open_circuit?: false,
+            circuit_until: nil
+          )
+        else
+          circuit_until = now + circuit_duration
+          Decision.new(
+            retry?: false,
+            wait: circuit_duration,
+            next_attempt: 0,
+            open_circuit?: true,
+            circuit_until: circuit_until
+          )
+        end
+      end
+
+      private
+
+      def policy_key
+        return :timeout if error.is_a?(SourceMonitor::Fetching::TimeoutError)
+        return :connection if error.is_a?(SourceMonitor::Fetching::ConnectionError)
+
+        if error.is_a?(SourceMonitor::Fetching::HTTPError)
+          status = error.status.to_i
+          return :http_429 if status == 429
+          return :http_5xx if status >= 500
+          return :http_4xx if status >= 400
+        end
+
+        return :parsing if error.is_a?(SourceMonitor::Fetching::ParsingError)
+        return :unexpected if error.is_a?(SourceMonitor::Fetching::UnexpectedResponseError)
+
+        :fallback
+      end
+    end
+  end
+end

data/lib/source_monitor/fetching/stalled_fetch_reconciler.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/numeric/time"
+
+module SourceMonitor
+  module Fetching
+    class StalledFetchReconciler
+      Result = Struct.new(:recovered_source_ids, :jobs_removed, :executed_at, keyword_init: true)
+
+      FAILURE_MESSAGE = "Fetch job stalled; resetting state and retrying"
+
+      def self.call(now: Time.current, stale_after: nil)
+        new(now:, stale_after: stale_after || default_stale_after).call
+      end
+
+      def initialize(now:, stale_after:)
+        @now = now
+        @stale_after = stale_after
+      end
+
+      def call
+        recovered_ids = []
+        removed_job_ids = []
+        jobs_supported = jobs_supported?
+
+        stale_sources.find_each do |source|
+          recovery = recover_source(source, jobs_supported:)
+          next if recovery.nil?
+
+          recovered_ids << recovery[:source_id] if recovery[:source_id]
+          removed_job_ids.concat(Array(recovery[:removed_job_ids]))
+        end
+
+        Result.new(
+          recovered_source_ids: recovered_ids.uniq,
+          jobs_removed: removed_job_ids.uniq,
+          executed_at: now
+        )
+      end
+
+      private
+
+      attr_reader :now, :stale_after
+
+      def self.default_stale_after
+        if defined?(SourceMonitor::Scheduler::STALE_QUEUE_TIMEOUT)
+          SourceMonitor::Scheduler::STALE_QUEUE_TIMEOUT
+        else
+          10.minutes
+        end
+      end
+
+      def stale_sources
+        cutoff = now - stale_after
+        SourceMonitor::Source.
+          where(fetch_status: "fetching").
+          where.not(last_fetch_started_at: nil).
+          where(SourceMonitor::Source.arel_table[:last_fetch_started_at].lteq(cutoff))
+      end
+
+      def recover_source(source, jobs_supported:)
+        removed_job_ids = []
+
+        source.with_lock do
+          source.reload
+          return nil unless stale?(source)
+
+          removed_job_ids = jobs_supported ? discard_jobs_for(source) : []
+          mark_source_failed!(source)
+          enqueue_recovery(source)
+
+          { source_id: source.id, removed_job_ids: removed_job_ids }
+        end
+      rescue StandardError => error
+        log_recovery_failure(source, error)
+        nil
+      end
+
+      def stale?(source)
+        source.fetch_status == "fetching" &&
+          source.last_fetch_started_at.present? &&
+          source.last_fetch_started_at <= now - stale_after
+      end
+
+      def discard_jobs_for(source)
+        return [] unless jobs_supported?
+
+        matching_jobs = jobs_for(source)
+        removed = []
+
+        matching_jobs.find_each do |job|
+          removed << job.id
+          if job.failed_execution.present?
+            job.failed_execution.discard
+          else
+            job.destroy!
+          end
+        end
+
+        removed
+      end
+
+      def jobs_for(source)
+        return ::SolidQueue::Job.none unless jobs_supported?
+
+        queue_name = SourceMonitor.queue_name(:fetch)
+        ::SolidQueue::Job.
+          where(queue_name: queue_name).
+          where("arguments::jsonb -> 'arguments' ->> 0 = ?", source.id.to_s)
+      end
+
+      def mark_source_failed!(source)
+        failure_attrs = {
+          fetch_status: "failed",
+          last_error: FAILURE_MESSAGE,
+          last_error_at: now,
+          failure_count: source.failure_count.to_i + 1,
+          next_fetch_at: now,
+          updated_at: now
+        }
+
+        source.update!(failure_attrs)
+        SourceMonitor::Realtime.broadcast_source(source) if SourceMonitor::Realtime.respond_to?(:broadcast_source)
+      end
+
+      def enqueue_recovery(source)
+        SourceMonitor::Fetching::FetchRunner.enqueue(source, force: true)
+      end
+
+      def log_recovery_failure(source, error)
+        return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+        Rails.logger.error(
+          "[SourceMonitor::Fetching::StalledFetchReconciler] Failed to recover source #{source&.id}: #{error.class}: #{error.message}"
+        )
+      end
+
+      def jobs_supported?
+        defined?(::SolidQueue::Job) &&
+          ::SolidQueue::Job.table_exists?
+      rescue ActiveRecord::StatementInvalid, ActiveRecord::NoDatabaseError
+        false
+      end
+    end
+  end
+end

data/lib/source_monitor/health/source_health_check.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Health
+    class SourceHealthCheck
+      Result = Struct.new(:log, :success?, :error, keyword_init: true)
+
+      def initialize(source:, client: nil, now: Time.current)
+        @source = source
+        @client = client
+        @now = now
+      end
+
+      def call
+        return Result.new(log: nil, success?: false, error: nil) unless source
+
+        started_at = now
+        response = nil
+        error = nil
+
+        begin
+          response = connection.get(source.feed_url)
+        rescue StandardError => exception
+          error = exception
+        end
+
+        completed_at = Time.current
+        log = create_log(response:, error:, started_at:, completed_at:)
+
+        Result.new(log:, success?: log&.success?, error: error)
+      end
+
+      private
+
+      attr_reader :source, :client, :now
+
+      def connection
+        @connection ||= (client || SourceMonitor::HTTP.client(headers: request_headers, retry_requests: false))
+      end
+
+      def request_headers
+        headers = (source.custom_headers || {}).each_with_object({}) do |(key, value), memo|
+          memo[key.to_s] = value
+        end
+
+        headers["If-None-Match"] = source.etag if source.etag.present?
+        headers["If-Modified-Since"] = source.last_modified.httpdate if source.last_modified.present?
+        headers
+      end
+
+      def create_log(response:, error:, started_at:, completed_at:)
+        attrs = {
+          source: source,
+          started_at: started_at,
+          completed_at: completed_at,
+          duration_ms: duration_ms(started_at, completed_at),
+          http_status: response_status(response, error),
+          http_response_headers: response_headers(response)
+        }
+
+        if error
+          attrs[:success] = false
+          attrs[:error_class] = error.class.name
+          attrs[:error_message] = error.message
+        else
+          attrs[:success] = successful_status?(response&.status)
+        end
+
+        SourceMonitor::HealthCheckLog.create!(attrs)
+      end
+
+      def duration_ms(started_at, completed_at)
+        ((completed_at - started_at) * 1000.0).round
+      end
+
+      def response_status(response, error)
+        return response.status if response&.respond_to?(:status)
+
+        if error.respond_to?(:response)
+          response_data = error.response
+          return response_data[:status] if response_data.is_a?(Hash)
+        end
+
+        nil
+      end
+
+      def response_headers(response)
+        return {} unless response&.respond_to?(:headers)
+
+        response.headers.each_with_object({}) do |(key, value), memo|
+          memo[key.to_s] = value
+        end
+      end
+
+      def successful_status?(status)
+        status.present? && status.to_i.between?(200, 399)
+      end
+    end
+  end
+end

data/lib/source_monitor/health/source_health_monitor.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/numeric/time"
+
+module SourceMonitor
+  module Health
+    class SourceHealthMonitor
+      attr_reader :source, :config, :now
+
+      def initialize(source:, config: SourceMonitor.config.health, now: Time.current)
+        @source = source
+        @config = config
+        @now = now
+      end
+
+      def call
+        reload_source
+
+        logs = recent_logs.to_a
+        return if logs.empty?
+
+        rate = calculate_success_rate(logs)
+        attrs = { rolling_success_rate: rate }
+        sample_size = logs.size
+        thresholds_active = thresholds_applicable?(sample_size)
+
+        auto_paused_until = current_auto_paused_until
+        auto_paused_at = current_auto_paused_at
+
+        if thresholds_active && should_resume?(auto_paused_until, rate)
+          auto_paused_until = nil
+          auto_paused_at = nil
+          attrs[:auto_paused_until] = nil
+          attrs[:auto_paused_at] = nil
+          attrs[:backoff_until] = nil if source.backoff_until.present?
+        end
+
+        if thresholds_active && should_auto_pause?(rate)
+          new_until = compute_auto_pause_until(auto_paused_until)
+          auto_paused_until = new_until
+          auto_paused_at ||= now
+
+          attrs[:auto_paused_until] = new_until
+          attrs[:auto_paused_at] = auto_paused_at
+          apply_backoff(attrs, new_until)
+        end
+
+        enforce_fixed_interval(attrs, auto_paused_until)
+
+        status = determine_status(rate, auto_paused_until, logs)
+        apply_status(attrs, status)
+
+        source.update!(attrs)
+      rescue ActiveRecord::RecordNotFound
+        # Source was deleted between fetch and health update.
+        nil
+      end
+
+      private
+
+      def reload_source
+        source.reload
+      end
+
+      def recent_logs
+        limit = [ config.window_size.to_i, 1 ].max
+        source.fetch_logs.order(started_at: :desc).limit(limit)
+      end
+
+      def calculate_success_rate(logs)
+        successes = logs.count { |log| log.success? }
+        total = logs.size
+        return 0.0 if total.zero?
+
+        (successes.to_f / total).round(4)
+      end
+
+      def current_auto_paused_until
+        source.auto_paused_until
+      end
+
+      def current_auto_paused_at
+        source.auto_paused_at
+      end
+
+      def should_resume?(auto_paused_until, rate)
+        return false if auto_paused_until.nil?
+
+        rate >= auto_resume_threshold
+      end
+
+      def should_auto_pause?(rate)
+        threshold = auto_pause_threshold
+        return false if threshold.nil?
+
+        rate < threshold
+      end
+
+      def compute_auto_pause_until(existing_until)
+        cooldown = [ config.auto_pause_cooldown_minutes.to_i, 1 ].max
+        proposed = now + cooldown.minutes
+
+        return proposed if existing_until.nil?
+        existing_until > proposed ? existing_until : proposed
+      end
+
+      def apply_backoff(attrs, pause_until)
+        if source.next_fetch_at.nil? || source.next_fetch_at < pause_until
+          attrs[:next_fetch_at] = pause_until
+        end
+
+        if source.backoff_until.nil? || source.backoff_until < pause_until
+          attrs[:backoff_until] = pause_until
+        end
+      end
+
+      def determine_status(rate, auto_paused_until, logs)
+        if auto_paused_active?(auto_paused_until)
+          "auto_paused"
+        elsif consecutive_failures(logs) >= 3
+          "declining"
+        elsif improving_streak?(logs)
+          "improving"
+        elsif rate >= healthy_threshold
+          "healthy"
+        elsif rate >= warning_threshold
+          "warning"
+        else
+          "critical"
+        end
+      end
+
+      def enforce_fixed_interval(attrs, auto_paused_until)
+        return if source.adaptive_fetching_enabled?
+        return if auto_paused_active?(auto_paused_until)
+
+        backoff_value = attrs.key?(:backoff_until) ? attrs[:backoff_until] : source.backoff_until
+        return if backoff_value.blank?
+
+        fixed_minutes = [ source.fetch_interval_minutes.to_i, 1 ].max
+        attrs[:next_fetch_at] = now + fixed_minutes.minutes
+        attrs[:backoff_until] = nil
+      end
+
+      def apply_status(attrs, status)
+        previous = source.health_status.presence || "healthy"
+        return if previous == status
+
+        attrs[:health_status] = status
+        attrs[:health_status_changed_at] = now
+      end
+
+      def auto_pause_threshold
+        value = source.health_auto_pause_threshold
+        value = config.auto_pause_threshold if value.nil?
+        value&.to_f
+      end
+
+      def auto_resume_threshold
+        [ config.auto_resume_threshold.to_f, auto_pause_threshold.to_f ].max
+      end
+
+      def healthy_threshold
+        [ config.healthy_threshold.to_f, warning_threshold ].max
+      end
+
+      def warning_threshold
+        config.warning_threshold.to_f
+      end
+
+      def auto_paused_active?(value)
+        value.present? && value.future?
+      end
+
+      def thresholds_applicable?(sample_size)
+        sample_size >= minimum_sample_size
+      end
+
+      def minimum_sample_size
+        [ config.window_size.to_i, 1 ].max
+      end
+
+      def consecutive_failures(logs)
+        logs.take_while { |log| !log_success?(log) }.size
+      end
+
+      def improving_streak?(logs)
+        success_streak = 0
+        failure_seen = false
+
+        logs.each do |log|
+          if log_success?(log)
+            success_streak += 1
+          else
+            failure_seen = true
+            break
+          end
+        end
+
+        success_streak >= 2 && failure_seen
+      end
+
+      def log_success?(log)
+        return log.success? if log.respond_to?(:success?)
+
+        !!log.success
+      end
+    end
+  end
+end