source_monitor 0.11.1 → 0.12.0

data/app/models/concerns/source_monitor/loggable.rb

@@ -13,6 +13,18 @@ module SourceMonitor
       scope :recent, -> { order(started_at: :desc) }
       scope :successful, -> { where(success: true) }
       scope :failed, -> { where(success: false) }
+      scope :since, ->(date) { where(arel_table[:started_at].gteq(date)) }
+      scope :before, ->(date) { where(arel_table[:started_at].lteq(date)) }
+      scope :today, -> { since(Time.current.beginning_of_day) }
+      scope :by_date_range, ->(start_date, end_date) { since(start_date).before(end_date) }
+
+      after_save :sync_log_entry
+    end
+
+    private
+
+    def sync_log_entry
+      SourceMonitor::Logs::EntrySync.call(self)
     end
   end
 end

data/app/models/source_monitor/fetch_log.rb

@@ -23,13 +23,5 @@ module SourceMonitor
     scope :by_category, ->(category) { where(error_category: category) }
 
     SourceMonitor::ModelExtensions.register(self, :fetch_log)
-
-    after_save :sync_log_entry
-
-    private
-
-    def sync_log_entry
-      SourceMonitor::Logs::EntrySync.call(self)
-    end
   end
 end

data/app/models/source_monitor/health_check_log.rb

@@ -16,13 +16,5 @@ module SourceMonitor
     validates :source, presence: true
 
     SourceMonitor::ModelExtensions.register(self, :health_check_log)
-
-    after_save :sync_log_entry
-
-    private
-
-    def sync_log_entry
-      SourceMonitor::Logs::EntrySync.call(self)
-    end
   end
 end

data/app/models/source_monitor/import_history.rb

@@ -2,7 +2,13 @@
 
 module SourceMonitor
   class ImportHistory < ApplicationRecord
+    attribute :imported_sources, default: -> { [] }
+    attribute :failed_sources, default: -> { [] }
+    attribute :skipped_duplicates, default: -> { [] }
+    attribute :bulk_settings, default: -> { {} }
+
     validates :user_id, presence: true
+    validate :completed_at_after_started_at
 
     scope :not_dismissed, -> { where(dismissed_at: nil) }
 
@@ -33,5 +39,13 @@ module SourceMonitor
 
       ((completed_at - started_at) * 1000).round
     end
+
+    private
+
+    def completed_at_after_started_at
+      return if completed_at.blank? || started_at.blank?
+
+      errors.add(:completed_at, "must be after started_at") if completed_at < started_at
+    end
   end
 end

data/app/models/source_monitor/import_session.rb

@@ -4,6 +4,8 @@ module SourceMonitor
   class ImportSession < ApplicationRecord
     STEP_ORDER = %w[upload preview health_check configure confirm].freeze
 
+    scope :in_step, ->(step) { where(current_step: step) }
+
     validates :current_step, inclusion: { in: STEP_ORDER }
     validates :user_id, presence: true
 

data/app/models/source_monitor/item.rb

@@ -24,6 +24,8 @@ module SourceMonitor
     validates :content_fingerprint, uniqueness: { scope: :source_id }, allow_blank: true
     validates :url, presence: true
 
+    after_create_commit :ensure_feed_content_record, if: -> { content.present? }
+
     scope :recent, -> { active.order(Arel.sql("published_at DESC NULLS LAST, created_at DESC")) }
     scope :published, -> { active.where.not(published_at: nil) }
     scope :pending_scrape, -> { active.where(scraped_at: nil) }
@@ -80,6 +82,19 @@ module SourceMonitor
       end
     end
 
+    def restore!
+      return unless deleted?
+
+      self.class.transaction do
+        update_columns(
+          deleted_at: nil,
+          updated_at: Time.current
+        )
+
+        SourceMonitor::Source.increment_counter(:items_count, source_id) if source_id
+      end
+    end
+
     private
 
     # Item content lives in a separate row that we only create when rich content exists.

data/app/models/source_monitor/item_content.rb

@@ -8,6 +8,8 @@ module SourceMonitor
 
     has_many_attached :images if defined?(ActiveStorage)
 
+    delegate :content, to: :item, prefix: :feed, allow_nil: true
+
     before_save :compute_word_counts
 
     SourceMonitor::ModelExtensions.register(self, :item_content)
@@ -30,11 +32,10 @@ module SourceMonitor
     end
 
     def compute_feed_word_count
-      content = item&.content
-      if content.blank?
+      if feed_content.blank?
         self.feed_word_count = nil
       else
-        stripped = ActionView::Base.full_sanitizer.sanitize(content)
+        stripped = ActionView::Base.full_sanitizer.sanitize(feed_content)
         self.feed_word_count = stripped.present? ? stripped.split.size : nil
       end
     end

data/app/models/source_monitor/scrape_log.rb

@@ -8,14 +8,16 @@ module SourceMonitor
     belongs_to :source, class_name: "SourceMonitor::Source", inverse_of: :scrape_logs
     has_one :log_entry, as: :loggable, class_name: "SourceMonitor::LogEntry", inverse_of: :loggable, dependent: :destroy
 
+    scope :by_source, ->(source) { where(source: source) }
+    scope :by_status, ->(success) { where(success: success) }
+    scope :by_item, ->(item) { where(item: item) }
+
     validates :item, :source, presence: true
     validates :content_length, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
     validate :source_matches_item
 
     SourceMonitor::ModelExtensions.register(self, :scrape_log)
 
-    after_save :sync_log_entry
-
     private
 
     def source_matches_item
@@ -23,9 +25,5 @@ module SourceMonitor
 
       errors.add(:source, "must match item source") if item.source_id != source_id
     end
-
-    def sync_log_entry
-      SourceMonitor::Logs::EntrySync.call(self)
-    end
   end
 end

data/app/models/source_monitor/source.rb

@@ -11,6 +11,7 @@ module SourceMonitor
     has_one_attached :favicon if defined?(ActiveStorage)
 
     FETCH_STATUS_VALUES = %w[idle queued fetching failed].freeze
+    HEALTH_STATUS_VALUES = %w[working declining improving failing].freeze
 
     has_many :all_items, class_name: "SourceMonitor::Item", inverse_of: :source, dependent: :destroy
     has_many :items, -> { active }, class_name: "SourceMonitor::Item", inverse_of: :source
@@ -28,6 +29,8 @@ module SourceMonitor
       where(failure.or(error_present).or(error_time_present))
     }
     scope :healthy, -> { active.where(failure_count: 0, last_error: nil, last_error_at: nil) }
+    scope :scraping_enabled, -> { where(scraping_enabled: true) }
+    scope :scraping_disabled, -> { where(scraping_enabled: false) }
 
     # Use Rails attribute API for default values instead of after_initialize callbacks
     attribute :scrape_settings, default: -> { {} }
@@ -48,6 +51,7 @@ module SourceMonitor
     validates :items_retention_days, numericality: { allow_nil: true, only_integer: true, greater_than_or_equal_to: 0 }
     validates :max_items, numericality: { allow_nil: true, only_integer: true, greater_than_or_equal_to: 0 }
     validates :fetch_status, inclusion: { in: FETCH_STATUS_VALUES }
+    validates :health_status, inclusion: { in: HEALTH_STATUS_VALUES }
     validates :fetch_retry_attempt, numericality: { greater_than_or_equal_to: 0, only_integer: true }
 
     validate :health_auto_pause_threshold_within_bounds
@@ -62,21 +66,20 @@ module SourceMonitor
       end
 
       def scrape_candidates(threshold: SourceMonitor.config.scraping.scrape_recommendation_threshold)
-        threshold_value = threshold.to_i
-        return none if threshold_value <= 0
-
-        active
-          .where(scraping_enabled: false)
-          .where(
-            "#{table_name}.id IN (
-              SELECT i.source_id
-              FROM #{Item.table_name} i
-              INNER JOIN #{ItemContent.table_name} ic ON ic.item_id = i.id
-              WHERE ic.feed_word_count IS NOT NULL
-              GROUP BY i.source_id
-              HAVING AVG(ic.feed_word_count) < ?
-            )", threshold_value
-          )
+        SourceMonitor::Queries::ScrapeCandidatesQuery.new(threshold:).call
+      end
+
+      # Bulk-enable scraping for sources that don't already have it enabled.
+      # Returns the number of records updated.
+      def enable_scraping!(ids)
+        default_adapter = column_defaults["scraper_adapter"] || "readability"
+
+        where(id: ids, scraping_enabled: false).update_all(
+          scraping_enabled: true,
+          auto_scrape: true,
+          scraper_adapter: default_adapter,
+          updated_at: Time.current
+        )
       end
 
       def ransackable_attributes(_auth_object = nil)
@@ -145,14 +148,20 @@ module SourceMonitor
 
     def reset_items_counter!
       # Recalculate items_count from actual active (non-deleted) items
-      actual_count = items.count
-      update_columns(items_count: actual_count)
+      # Use reset_counters which is the Rails-native way to fix counter caches
+      self.class.reset_counters(id, :items)
+      reload
+    end
+
+    def clear_favicon_cooldown!
+      metadata_without_cooldown = (metadata || {}).except("favicon_last_attempted_at")
+      update_column(:metadata, metadata_without_cooldown)
     end
 
     def avg_word_count
       items.joins(:item_content)
-           .where.not(sourcemon_item_contents: { scraped_word_count: nil })
-           .average("sourcemon_item_contents.scraped_word_count")
+           .where.not(ItemContent.table_name => { scraped_word_count: nil })
+           .average("#{ItemContent.table_name}.scraped_word_count")
            &.round
     end
 

data/app/presenters/source_monitor/base_presenter.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class BasePresenter < SimpleDelegator
+    include ActionView::Helpers::NumberHelper
+    include ActionView::Helpers::DateHelper
+    include ActionView::Helpers::TextHelper
+
+    def initialize(model)
+      super(model)
+    end
+
+    def model
+      __getobj__
+    end
+
+    alias_method :object, :model
+  end
+end

data/app/presenters/source_monitor/source_details_presenter.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class SourceDetailsPresenter < BasePresenter
+    DATE_FORMAT = "%b %d, %Y %H:%M %Z"
+
+    def fetch_interval_display
+      hours = number_with_precision(fetch_interval_minutes / 60.0, precision: 2)
+      "#{fetch_interval_minutes} minutes (~#{hours} hours)"
+    end
+
+    def circuit_state_label
+      if fetch_circuit_open?
+        until_time = fetch_circuit_until&.strftime(DATE_FORMAT) || "unknown"
+        "Open until #{until_time}"
+      else
+        "Closed"
+      end
+    end
+
+    def adaptive_interval_label
+      adaptive_fetching_enabled? ? "Auto" : "Fixed"
+    end
+
+    def formatted_next_fetch_at
+      format_timestamp(next_fetch_at)
+    end
+
+    def formatted_last_fetched_at
+      format_timestamp(last_fetched_at)
+    end
+
+    def details_hash
+      {
+        "Fetch interval" => fetch_interval_display,
+        "Adaptive interval" => adaptive_interval_label,
+        "Scraper" => scraper_adapter,
+        "Feed content" => feed_content_readability_enabled? ? "Readability" : "Raw",
+        "Active" => active? ? "Yes" : "No",
+        "Scraping" => scraping_enabled? ? "Enabled" : "Disabled",
+        "Auto scrape" => auto_scrape? ? "Enabled" : "Disabled",
+        "Requires JS" => requires_javascript? ? "Yes" : "No",
+        "Failure count" => failure_count,
+        "Retry attempt" => fetch_retry_attempt,
+        "Circuit state" => circuit_state_label,
+        "Last error" => last_error.presence || "None",
+        "Items count" => items_count,
+        "Retention days" => items_retention_days || "\u2014",
+        "Max items" => max_items || "\u2014"
+      }
+    end
+
+    private
+
+    def format_timestamp(time)
+      return "\u2014" if time.nil?
+
+      time.strftime(DATE_FORMAT)
+    end
+  end
+end

data/app/presenters/source_monitor/sources_filter_presenter.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  # Encapsulates filter state logic for the sources index view.
+  # Extracts filter detection, label generation, and active-filter
+  # tracking from the template into a testable object.
+  class SourcesFilterPresenter
+    DROPDOWN_FILTER_KEYS = %w[
+      active_eq
+      health_status_eq
+      feed_format_eq
+      scraper_adapter_eq
+      scraping_enabled_eq
+      avg_feed_words_lt
+    ].freeze
+
+    attr_reader :search_params, :search_term, :fetch_interval_filter, :adapter_options
+
+    # @param search_params [Hash] sanitized Ransack search params
+    # @param search_term [String] current text search term
+    # @param fetch_interval_filter [Hash, nil] active fetch interval filter
+    # @param adapter_options [Array<String>] distinct scraper adapter values
+    def initialize(search_params:, search_term:, fetch_interval_filter:, adapter_options: [])
+      @search_params = search_params || {}
+      @search_term = search_term.to_s
+      @fetch_interval_filter = fetch_interval_filter
+      @adapter_options = adapter_options
+    end
+
+    def has_any_filter?
+      @search_term.present? || @fetch_interval_filter.present? || active_filter_keys.any?
+    end
+
+    # Returns the subset of DROPDOWN_FILTER_KEYS that have present values
+    def active_filter_keys
+      DROPDOWN_FILTER_KEYS.select { |k| @search_params[k].present? }
+    end
+
+    # Returns a hash mapping active filter keys to human-readable labels
+    def filter_labels
+      {
+        "active_eq" => status_label,
+        "health_status_eq" => "Health: #{@search_params['health_status_eq']&.titleize}",
+        "feed_format_eq" => "Format: #{@search_params['feed_format_eq']&.upcase}",
+        "scraper_adapter_eq" => "Adapter: #{@search_params['scraper_adapter_eq']&.titleize}",
+        "scraping_enabled_eq" => scraping_label,
+        "avg_feed_words_lt" => "Avg Feed Words: < #{@search_params['avg_feed_words_lt']}"
+      }
+    end
+
+    private
+
+    def status_label
+      @search_params["active_eq"] == "true" ? "Status: Active" : "Status: Paused"
+    end
+
+    def scraping_label
+      @search_params["scraping_enabled_eq"] == "true" ? "Scraping: Enabled" : "Scraping: Disabled"
+    end
+  end
+end

data/app/views/source_monitor/dashboard/_recent_activity.html.erb

@@ -32,9 +32,9 @@
             <% end %>
           </div>
           <div class="text-right">
-            <span class="inline-flex items-center rounded-full px-3 py-1 text-xs font-semibold <%= event[:status] == :success ? "bg-green-100 text-green-700" : "bg-rose-100 text-rose-700" %>">
-              <%= event[:status] == :success ? "Success" : "Failure" %>
-            </span>
+            <%= render SourceMonitor::StatusBadgeComponent.new(
+                  status: event[:status] == :success ? "success" : "failed",
+                  label: event[:status] == :success ? "Success" : "Failure") %>
             <div class="mt-1 text-xs text-slate-400">
               <%= event[:time]&.strftime("%b %d, %H:%M") || "Unknown" %>
             </div>

data/app/views/source_monitor/dashboard/_stat_card.html.erb

@@ -1,5 +1,6 @@
 <% value = (stat_card[:value] || 0) %>
-<div class="rounded-lg border border-slate-200 bg-white p-5 shadow-sm">
+<% card_id = stat_card[:key] ? "source_monitor_stat_#{stat_card[:key]}" : nil %>
+<div id="<%= card_id %>" class="rounded-lg border border-slate-200 bg-white p-5 shadow-sm">
   <dt class="text-xs font-medium uppercase tracking-wide text-slate-500"><%= stat_card[:label] %></dt>
   <dd class="mt-2 text-3xl font-semibold text-slate-900"><%= value %></dd>
   <p class="mt-1 text-xs text-slate-500"><%= stat_card[:caption] %></p>

data/app/views/source_monitor/dashboard/_stats.html.erb

@@ -1,12 +1,10 @@
 <div id="source_monitor_dashboard_stats">
   <div class="grid gap-5 sm:grid-cols-2 xl:grid-cols-5">
-    <%= render partial: "stat_card", collection: [
-      { label: "Sources", value: stats[:total_sources], caption: "Total registered" },
-      { label: "Active", value: stats[:active_sources], caption: "Fetching on schedule" },
-      { label: "Failures", value: stats[:failed_sources], caption: "Require attention" },
-      { label: "Items", value: stats[:total_items], caption: "Stored entries" },
-      { label: "Fetches Today", value: stats[:fetches_today], caption: "Completed runs" }
-    ], locals: { stats: stats } %>
+    <%= render partial: "stat_card", locals: { stat_card: { key: "total_sources", label: "Sources", value: stats[:total_sources], caption: "Total registered" } } %>
+    <%= render partial: "stat_card", locals: { stat_card: { key: "active_sources", label: "Active", value: stats[:active_sources], caption: "Fetching on schedule" } } %>
+    <%= render partial: "stat_card", locals: { stat_card: { key: "failed_sources", label: "Failures", value: stats[:failed_sources], caption: "Require attention" } } %>
+    <%= render partial: "stat_card", locals: { stat_card: { key: "total_items", label: "Items", value: stats[:total_items], caption: "Stored entries" } } %>
+    <%= render partial: "stat_card", locals: { stat_card: { key: "fetches_today", label: "Fetches Today", value: stats[:fetches_today], caption: "Completed runs" } } %>
   </div>
 
   <% if stats[:health_distribution]&.values&.any? { |v| v > 0 } %>

data/app/views/source_monitor/items/_details.html.erb

@@ -3,22 +3,18 @@
 <% manual_scrape_disabled = !source&.scraping_enabled? %>
 <% scrape_badge = item_scrape_status_badge(item: item, source: source) %>
 <% inflight_scrape = %w[pending processing].include?(scrape_badge[:status]) %>
-<% recent_scrape_logs = item.scrape_logs.order(started_at: :desc).limit(5) %>
-<% latest_scrape_log = recent_scrape_logs.first %>
+<% recent_scrape_logs = local_assigns[:recent_scrape_logs] || @recent_scrape_logs || [] %>
+<% latest_scrape_log = local_assigns[:latest_scrape_log] || @latest_scrape_log || recent_scrape_logs.first %>
 
 <div class="space-y-8">
   <div class="flex flex-col justify-between gap-4 sm:flex-row sm:items-start">
     <div>
       <h1 class="text-3xl font-semibold text-slate-900"><%= item.title.presence || "(untitled)" %></h1>
       <div class="mt-2 flex flex-wrap items-center gap-2 text-xs">
-        <span
-          data-testid="scrape-status-badge"
-          data-status="<%= scrape_badge[:status] %>"
-          class="inline-flex items-center gap-1 rounded-full px-3 py-1 font-semibold <%= scrape_badge[:classes] %>"
-        >
-          <%= loading_spinner_svg if scrape_badge[:show_spinner] %>
-          <%= scrape_badge[:label] %>
-        </span>
+        <%= render SourceMonitor::StatusBadgeComponent.new(
+              status: scrape_badge[:status],
+              label: scrape_badge[:label],
+              data: { testid: "scrape-status-badge" }) %>
         <% if item.scraped_at.present? %>
           <span class="rounded-full bg-slate-100 px-3 py-1 font-semibold text-slate-600">
             Last scraped <%= time_ago_in_words(item.scraped_at) %> ago
@@ -110,9 +106,10 @@
                 <% recent_scrape_logs.first(3).each do |log| %>
                   <li class="flex items-center justify-between">
                     <span><%= log.started_at&.strftime("%b %d, %H:%M") || "Unknown" %></span>
-                    <span class="ml-3 inline-flex items-center rounded-full px-2 py-0.5 font-semibold <%= log.success? ? "bg-green-100 text-green-700" : "bg-rose-100 text-rose-700" %>">
-                      <%= log.success? ? "Success" : "Failure" %>
-                    </span>
+                    <%= render SourceMonitor::StatusBadgeComponent.new(
+                          status: log.success? ? "success" : "failed",
+                          label: log.success? ? "Success" : "Failure",
+                          size: :sm) %>
                   </li>
                 <% end %>
               </ul>
@@ -162,7 +159,7 @@
               <%= link_to "View Original", item.url, target: "_blank", rel: "noopener", class: "inline-flex items-center rounded-md bg-blue-600 px-3 py-2 text-sm font-medium text-white shadow hover:bg-blue-500" %>
             <% end %>
             <%= button_to "Manual Scrape",
-                  source_monitor.scrape_item_path(item),
+                  source_monitor.item_scrape_path(item),
                   method: :post,
                   class: [
                     "inline-flex items-center rounded-md px-3 py-2 text-sm font-medium shadow",

data/app/views/source_monitor/items/index.html.erb

@@ -100,18 +100,11 @@
                 <% end %>
               </td>
               <td class="px-6 py-4 text-xs">
-                <% status_label, status_classes =
-                     case item.scrape_status
-                     when "success"
-                       ["Scraped", "bg-green-100 text-green-700"]
-                     when "failed"
-                       ["Failed", "bg-rose-100 text-rose-700"]
-                     when "pending"
-                       ["Pending", "bg-amber-100 text-amber-700"]
-                     else
-                       ["Not Scraped", "bg-slate-100 text-slate-600"]
-                     end %>
-                <span class="inline-flex items-center rounded-full px-3 py-1 font-semibold <%= status_classes %>"><%= status_label %></span>
+                <% badge = item_scrape_status_badge(item: item, source: @source, show_spinner: false) %>
+                <%= render SourceMonitor::StatusBadgeComponent.new(
+                      status: badge[:status],
+                      label: badge[:label],
+                      show_spinner: false) %>
               </td>
               <td class="px-6 py-4 text-xs text-slate-500">
                 <%= item.item_content&.feed_word_count || "\u2014" %>
@@ -131,29 +124,11 @@
           <% end %>
         </tbody>
       </table>
-      <div class="flex flex-col items-center gap-3 border-t border-slate-200 px-6 py-4 sm:flex-row sm:justify-between">
-        <div class="text-xs text-slate-500">
-          Page <%= @page %>
-        </div>
-        <div class="flex gap-2">
-          <% prev_params = { page: @page - 1 } %>
-          <% prev_params[:q] = @search_params if @search_params.present? %>
-          <% next_params = { page: @page + 1 } %>
-          <% next_params[:q] = @search_params if @search_params.present? %>
-
-          <% if @has_previous_page %>
-            <%= link_to "Previous", source_monitor.items_path(prev_params), class: "inline-flex items-center rounded-md border border-slate-300 px-3 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50", data: { turbo_frame: "source_monitor_items_table" } %>
-          <% else %>
-            <span class="inline-flex items-center rounded-md border border-slate-200 px-3 py-2 text-sm font-medium text-slate-300">Previous</span>
-          <% end %>
-
-          <% if @has_next_page %>
-            <%= link_to "Next", source_monitor.items_path(next_params), class: "inline-flex items-center rounded-md border border-slate-300 px-3 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50", data: { turbo_frame: "source_monitor_items_table" } %>
-          <% else %>
-            <span class="inline-flex items-center rounded-md border border-slate-200 px-3 py-2 text-sm font-medium text-slate-300">Next</span>
-          <% end %>
-        </div>
-      </div>
+      <%= render "source_monitor/shared/pagination",
+            paginator_result: @paginator,
+            base_path: source_monitor.items_path,
+            extra_params: @search_params.present? ? { q: @search_params } : {},
+            turbo_frame: "source_monitor_items_table" %>
     <% end %>
   </div>
 </div>