source_monitor 0.10.2 → 0.11.0

data/app/assets/javascripts/source_monitor/controllers/modal_controller.js

@@ -3,9 +3,13 @@ import { Controller } from "@hotwired/stimulus";
 export default class extends Controller {
   static targets = ["panel"];
   static classes = ["open"];
+  static values = { autoOpen: Boolean, removeOnClose: Boolean };
 
   connect() {
     this.handleEscape = this.handleEscape.bind(this);
+    if (this.autoOpenValue) {
+      this.open();
+    }
   }
 
   disconnect() {
@@ -35,6 +39,10 @@ export default class extends Controller {
     }
 
     this.teardown();
+
+    if (this.removeOnCloseValue) {
+      this.element.remove();
+    }
   }
 
   backdrop(event) {

data/app/assets/javascripts/source_monitor/controllers/select_all_controller.js

@@ -1,18 +1,21 @@
 import { Controller } from "@hotwired/stimulus";
 
 export default class extends Controller {
-  static targets = ["master", "item"];
+  static targets = ["master", "item", "actionBar", "count"];
 
   connect() {
     this.syncMaster();
+    this.updateActionBar();
   }
 
   itemTargetConnected() {
     this.syncMaster();
+    this.updateActionBar();
   }
 
   itemTargetDisconnected() {
     this.syncMaster();
+    this.updateActionBar();
   }
 
   toggleAll(event) {
@@ -21,16 +24,33 @@ export default class extends Controller {
       if (checkbox.disabled) return;
       checkbox.checked = checked;
     });
+    this.updateActionBar();
   }
 
   toggleItem() {
     this.syncMaster();
+    this.updateActionBar();
   }
 
   syncMaster() {
     if (!this.hasMasterTarget) return;
     const selectable = this.itemTargets.filter((checkbox) => !checkbox.disabled);
-    const allChecked = selectable.length > 0 && selectable.every((checkbox) => checkbox.checked);
+    const allChecked =
+      selectable.length > 0 &&
+      selectable.every((checkbox) => checkbox.checked);
     this.masterTarget.checked = allChecked;
   }
+
+  updateActionBar() {
+    if (!this.hasActionBarTarget) return;
+    const checkedCount = this.itemTargets.filter((cb) => cb.checked).length;
+    if (this.hasCountTarget) {
+      this.countTarget.textContent = checkedCount;
+    }
+    if (checkedCount > 0) {
+      this.actionBarTarget.classList.remove("hidden");
+    } else {
+      this.actionBarTarget.classList.add("hidden");
+    }
+  }
 }

data/app/assets/stylesheets/source_monitor/application.tailwind.css

@@ -7,7 +7,7 @@
     @apply min-h-screen bg-slate-50 text-slate-900;
   }
 
-  .fm-admin a {
+  .fm-admin a:not([class*="bg-"]) {
     @apply text-blue-600 hover:text-blue-500;
   }
 }

data/app/controllers/source_monitor/bulk_scrape_enablements_controller.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class BulkScrapeEnablementsController < ApplicationController
+    def create
+      source_ids = Array(params.dig(:bulk_scrape_enablement, :source_ids)).map(&:to_i).reject(&:zero?)
+
+      if source_ids.empty?
+        handle_empty_selection
+        return
+      end
+
+      sources = Source.where(id: source_ids, scraping_enabled: false)
+      updated_count = sources.update_all(
+        scraping_enabled: true,
+        auto_scrape: true,
+        scraper_adapter: default_adapter,
+        updated_at: Time.current
+      )
+
+      respond_to do |format|
+        format.turbo_stream do
+          responder = SourceMonitor::TurboStreams::StreamResponder.new
+          responder.toast(
+            message: "Scraping enabled for #{updated_count} #{'source'.pluralize(updated_count)}.",
+            level: :success
+          )
+          responder.redirect(source_monitor.sources_path)
+          render turbo_stream: responder.render(view_context)
+        end
+        format.html do
+          redirect_to source_monitor.sources_path,
+            notice: "Scraping enabled for #{updated_count} #{'source'.pluralize(updated_count)}."
+        end
+      end
+    end
+
+    private
+
+    def default_adapter
+      "readability"
+    end
+
+    def handle_empty_selection
+      respond_to do |format|
+        format.turbo_stream do
+          responder = SourceMonitor::TurboStreams::StreamResponder.new
+          responder.toast(message: "No sources selected.", level: :warning)
+          render turbo_stream: responder.render(view_context), status: :unprocessable_entity
+        end
+        format.html do
+          redirect_to source_monitor.sources_path, alert: "No sources selected."
+        end
+      end
+    end
+  end
+end

data/app/controllers/source_monitor/dashboard_controller.rb

@@ -17,11 +17,20 @@ module SourceMonitor
       ).to_a
       @job_adapter = SourceMonitor::Jobs::Visibility.adapter_name
       @job_metrics = queries.job_metrics
-      fetch_schedule = queries.upcoming_fetch_schedule
+      @schedule_pages = schedule_pages_params
+      fetch_schedule = queries.upcoming_fetch_schedule(pages: @schedule_pages)
       @fetch_schedule_groups = fetch_schedule.groups
       @fetch_schedule_reference_time = fetch_schedule.reference_time
+      @scrape_candidates_count = @stats[:scrape_candidates_count]
+      @scrape_recommendation_threshold = SourceMonitor.config.scraping.scrape_recommendation_threshold
       @mission_control_enabled = SourceMonitor.mission_control_enabled?
       @mission_control_dashboard_path = SourceMonitor.mission_control_dashboard_path
     end
+
+    private
+
+    def schedule_pages_params
+      params.fetch(:schedule_pages, {}).permit!.to_h
+    end
   end
 end

data/app/controllers/source_monitor/import_history_dismissals_controller.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class ImportHistoryDismissalsController < ApplicationController
+    def create
+      import_history = ImportHistory.find(params[:import_history_id])
+      import_history.update!(dismissed_at: Time.current)
+
+      respond_to do |format|
+        format.turbo_stream do
+          render turbo_stream: turbo_stream.remove("source_monitor_import_history_panel")
+        end
+
+        format.html do
+          redirect_to source_monitor.sources_path, notice: "Import dismissed"
+        end
+      end
+    end
+  end
+end

data/app/controllers/source_monitor/source_retries_controller.rb

@@ -7,8 +7,16 @@ module SourceMonitor
     before_action :set_source
 
     def create
-      SourceMonitor::Fetching::FetchRunner.enqueue(@source.id, force: true)
-      render_fetch_enqueue_response("Retry has been forced and will run shortly.")
+      result = SourceMonitor::Fetching::FetchRunner.enqueue(@source.id, force: true)
+
+      if result == :already_fetching
+        render_fetch_enqueue_response(
+          "Fetch already in progress for this source. Please wait for the current fetch to complete.",
+          toast_level: :warning
+        )
+      else
+        render_fetch_enqueue_response("Retry has been forced and will run shortly.")
+      end
     rescue StandardError => error
       handle_fetch_failure(error)
     end

data/app/controllers/source_monitor/source_scrape_tests_controller.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class SourceScrapeTestsController < ApplicationController
+    before_action :set_source
+
+    def create
+      item = pick_test_item
+      unless item
+        handle_no_item
+        return
+      end
+
+      result = SourceMonitor::Scraping::ItemScraper.new(item: item, source: @source).call
+
+      @test_result = {
+        item: item.reload,
+        scrape_result: result,
+        feed_word_count: item.item_content&.feed_word_count,
+        scraped_word_count: item.item_content&.scraped_word_count,
+        feed_content_preview: item.content.to_s.truncate(500),
+        scraped_content_preview: item.item_content&.scraped_content.to_s.truncate(500),
+        improvement: compute_improvement(item)
+      }
+
+      respond_to do |format|
+        format.turbo_stream do
+          render turbo_stream: [
+            turbo_stream.remove("scrape_test_modal_#{@source.id}"),
+            turbo_stream.append_all("body",
+              partial: "source_monitor/source_scrape_tests/result",
+              locals: { source: @source, test_result: @test_result })
+          ]
+        end
+        format.html { render :show }
+      end
+    end
+
+    private
+
+    def set_source
+      @source = Source.find(params[:source_id])
+    end
+
+    def pick_test_item
+      @source.items
+             .joins(:item_content)
+             .where.not(sourcemon_item_contents: { feed_word_count: nil })
+             .order(published_at: :desc)
+             .first
+    end
+
+    def handle_no_item
+      respond_to do |format|
+        format.turbo_stream do
+          responder = SourceMonitor::TurboStreams::StreamResponder.new
+          responder.toast(message: "No items with feed content available for test scrape.", level: :warning)
+          render turbo_stream: responder.render(view_context)
+        end
+        format.html do
+          redirect_to source_monitor.source_path(@source), alert: "No items available for test scrape."
+        end
+      end
+    end
+
+    def compute_improvement(item)
+      feed = item.item_content&.feed_word_count.to_i
+      scraped = item.item_content&.scraped_word_count.to_i
+      return 0 if feed.zero?
+      ((scraped - feed).to_f / feed * 100).round(1)
+    end
+  end
+end

data/app/controllers/source_monitor/sources_controller.rb

@@ -20,29 +20,27 @@ module SourceMonitor
 
     def index
       @search_params = sanitized_search_params
-      @q = build_search_query
+      expand_scrape_recommendation_filter
+      @q = build_search_query(params: @search_params)
 
-      paginator = SourceMonitor::Pagination::Paginator.new(
+      @paginator = SourceMonitor::Pagination::Paginator.new(
         scope: @q.result,
         page: params[:page],
         per_page: params[:per_page] || PER_PAGE
       ).paginate
 
-      @sources = paginator.records
-      @page = paginator.page
-      @has_next_page = paginator.has_next_page
-      @has_previous_page = paginator.has_previous_page
+      @sources = @paginator.records
 
       @search_term = @search_params[SEARCH_FIELD.to_s].to_s.strip
       @search_field = SEARCH_FIELD
 
       metrics = SourceMonitor::Analytics::SourcesIndexMetrics.new(
         base_scope: Source.all,
-        result_scope: paginator.records,
+        result_scope: @paginator.records,
         search_params: @search_params
       )
 
-      @recent_import_histories = SourceMonitor::ImportHistory.recent_for(source_monitor_current_user&.id).limit(5)
+      @recent_import_histories = SourceMonitor::ImportHistory.not_dismissed.recent_for(source_monitor_current_user&.id).limit(5)
 
       @fetch_interval_distribution = metrics.fetch_interval_distribution
       @fetch_interval_filter = metrics.fetch_interval_filter
@@ -62,6 +60,8 @@ module SourceMonitor
         @avg_feed_word_counts = {}
         @avg_scraped_word_counts = {}
       end
+
+      @scrape_candidate_ids = compute_scrape_candidate_ids
     end
 
     def show
@@ -90,8 +90,17 @@ module SourceMonitor
     end
 
     def update
+      scraping_was_disabled = !@source.scraping_enabled?
+
       if @source.update(source_params)
-        redirect_to source_monitor.source_path(@source), notice: "Source updated successfully"
+        notice = "Source updated successfully"
+
+        if scraping_was_disabled && @source.scraping_enabled?
+          enqueued = enqueue_unscraped_items(@source)
+          notice = "Auto-scraping enabled. #{enqueued} existing #{'item'.pluralize(enqueued)} queued for scraping."
+        end
+
+        redirect_to source_monitor.source_path(@source), notice: notice
       else
         render :edit, status: :unprocessable_entity
       end
@@ -180,6 +189,39 @@ module SourceMonitor
       end
     end
 
+    def expand_scrape_recommendation_filter
+      return unless @search_params["scraping_enabled_eq"] == "recommend"
+
+      threshold = SourceMonitor.config.scraping.scrape_recommendation_threshold
+      @search_params.delete("scraping_enabled_eq")
+      @search_params["scraping_enabled_eq"] = "false"
+      @search_params["active_eq"] = "true"
+      @search_params["avg_feed_words_lt"] = threshold.to_s
+    end
+
+    def compute_scrape_candidate_ids
+      threshold = SourceMonitor.config.scraping.scrape_recommendation_threshold
+      return Set.new if threshold.nil? || threshold <= 0
+
+      candidate_ids = @sources.select do |source|
+        avg = @avg_feed_word_counts[source.id]
+        avg.present? && avg < threshold && !source.scraping_enabled?
+      end.map(&:id)
+
+      Set.new(candidate_ids)
+    end
+
+    def enqueue_unscraped_items(source)
+      result = SourceMonitor::Scraping::BulkSourceScraper.new(
+        source: source,
+        selection: :unscraped
+      ).call
+      result.enqueued_count
+    rescue StandardError => error
+      Rails.logger.warn("[SourceMonitor] Failed to enqueue unscraped items: #{error.message}") if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+      0
+    end
+
     def enqueue_favicon_fetch(source)
       return unless defined?(ActiveStorage)
       return unless SourceMonitor.config.favicons.enabled?

data/app/helpers/source_monitor/application_helper.rb

@@ -249,6 +249,30 @@ module SourceMonitor
       end
     end
 
+    def pagination_page_numbers(current_page:, total_pages:, window: 2)
+      return [ 1 ] if total_pages <= 1
+
+      # When total pages fit without gaps, show them all
+      if total_pages <= (2 * window) + 3
+        return (1..total_pages).to_a
+      end
+
+      pages = [ 1, total_pages ]
+      ((current_page - window)..(current_page + window)).each do |p|
+        pages << p if p >= 1 && p <= total_pages
+      end
+      pages = pages.uniq.sort
+
+      result = []
+      last = 0
+      pages.each do |p|
+        result << :gap if p > last + 1
+        result << p
+        last = p
+      end
+      result
+    end
+
     private
 
     def external_link_icon

data/app/helpers/source_monitor/health_badge_helper.rb

@@ -5,15 +5,13 @@ module SourceMonitor
     def source_health_badge(source, override: nil)
       return override if override.present?
 
-      status = source&.health_status.presence || "healthy"
+      status = source&.health_status.presence || "working"
 
       mapping = {
-        "healthy" => { label: "Healthy", classes: "bg-green-100 text-green-700", show_spinner: false },
-        "warning" => { label: "Needs Attention", classes: "bg-amber-100 text-amber-700", show_spinner: false },
-        "critical" => { label: "Failing", classes: "bg-rose-100 text-rose-700", show_spinner: false },
-        "declining" => { label: "Declining", classes: "bg-orange-100 text-orange-700", show_spinner: false },
+        "working" => { label: "Working", classes: "bg-green-100 text-green-700", show_spinner: false },
+        "declining" => { label: "Declining", classes: "bg-yellow-100 text-yellow-700", show_spinner: false },
         "improving" => { label: "Improving", classes: "bg-sky-100 text-sky-700", show_spinner: false },
-        "auto_paused" => { label: "Auto-Paused", classes: "bg-amber-100 text-amber-700", show_spinner: false },
+        "failing" => { label: "Failing", classes: "bg-rose-100 text-rose-700", show_spinner: false },
         "unknown" => { label: "Unknown", classes: "bg-slate-100 text-slate-600", show_spinner: false }
       }
 
@@ -21,11 +19,11 @@ module SourceMonitor
     end
 
     def source_health_actions(source)
-      status = source&.health_status.presence || "healthy"
+      status = source&.health_status.presence || "working"
       helpers = SourceMonitor::Engine.routes.url_helpers
 
       case status
-      when "critical", "declining"
+      when "failing", "declining"
         [
           {
             key: :full_fetch,
@@ -44,17 +42,6 @@ module SourceMonitor
             data: { testid: "source-health-action-health_check" }
           }
         ]
-      when "auto_paused"
-        [
-          {
-            key: :reset,
-            label: "Reset to Active Status",
-            description: "Clears the pause window, failure counters, and schedules the next fetch using the configured interval.",
-            path: helpers.source_health_reset_path(source),
-            method: :post,
-            data: { testid: "source-health-action-reset" }
-          }
-        ]
       else
         []
       end
@@ -63,7 +50,7 @@ module SourceMonitor
     def interactive_health_status?(source, override: nil)
       return false if override.present?
 
-      %w[critical declining auto_paused].include?(source&.health_status.presence)
+      %w[failing declining].include?(source&.health_status.presence)
     end
   end
 end

data/app/jobs/source_monitor/fetch_feed_job.rb

@@ -7,15 +7,22 @@ module SourceMonitor
 
     source_monitor_queue :fetch
 
+    SCHEDULED_CONCURRENCY_MAX_ATTEMPTS = 5
+
     discard_on ActiveJob::DeserializationError
-    retry_on SourceMonitor::Fetching::FetchRunner::ConcurrencyError,
-             wait: FETCH_CONCURRENCY_RETRY_WAIT,
-             attempts: 5
+
+    rescue_from SourceMonitor::Fetching::FetchRunner::ConcurrencyError do |error|
+      handle_concurrency_error(error)
+    end
 
     def perform(source_id, force: false)
+      @source_id = source_id
+      @force = force
+
       source = SourceMonitor::Source.find_by(id: source_id)
       return unless source
 
+      @source = source
       return unless should_run?(source, force: force)
 
       SourceMonitor::Fetching::FetchRunner.new(source: source, force: force).run
@@ -25,6 +32,28 @@ module SourceMonitor
 
     private
 
+    def handle_concurrency_error(error)
+      if @force
+        log_force_fetch_skipped
+        @source&.update_columns(fetch_status: "idle") if @source&.fetch_status == "queued"
+      else
+        attempt = executions
+        if attempt < SCHEDULED_CONCURRENCY_MAX_ATTEMPTS
+          retry_job wait: FETCH_CONCURRENCY_RETRY_WAIT
+        else
+          raise error
+        end
+      end
+    end
+
+    def log_force_fetch_skipped
+      return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+      Rails.logger.info("[SourceMonitor::FetchFeedJob] Fetch already in progress for source #{@source_id}, skipping force-fetch")
+    rescue StandardError
+      nil
+    end
+
     def should_run?(source, force:)
       return true if force
 

data/app/jobs/source_monitor/source_health_check_job.rb

@@ -24,7 +24,7 @@ module SourceMonitor
       nil
     end
 
-    DEGRADED_STATUSES = %w[declining critical warning].freeze
+    DEGRADED_STATUSES = %w[declining failing].freeze
 
     private
 

data/app/models/source_monitor/fetch_log.rb

@@ -12,11 +12,15 @@ module SourceMonitor
     attribute :items_failed, :integer, default: 0
     attribute :http_response_headers, default: -> { {} }
 
+    ERROR_CATEGORIES = %w[network parse blocked auth unknown].freeze
+
     validates :source, presence: true
     validates :items_created, :items_updated, :items_failed,
               numericality: { greater_than_or_equal_to: 0 }
+    validates :error_category, inclusion: { in: ERROR_CATEGORIES }, allow_nil: true
 
     scope :for_job, ->(job_id) { where(job_id:) }
+    scope :by_category, ->(category) { where(error_category: category) }
 
     SourceMonitor::ModelExtensions.register(self, :fetch_log)
 

data/app/models/source_monitor/import_history.rb

@@ -4,6 +4,8 @@ module SourceMonitor
   class ImportHistory < ApplicationRecord
     validates :user_id, presence: true
 
+    scope :not_dismissed, -> { where(dismissed_at: nil) }
+
     scope :recent_for, lambda { |user_id|
       scope = order(created_at: :desc)
       scope = scope.where(user_id: user_id) if user_id

data/app/models/source_monitor/source.rb

@@ -34,7 +34,7 @@ module SourceMonitor
     attribute :custom_headers, default: -> { {} }
     attribute :metadata, default: -> { {} }
     attribute :fetch_status, :string, default: "idle"
-    attribute :health_status, :string, default: "healthy"
+    attribute :health_status, :string, default: "working"
 
     sanitizes_string_attributes :name, :feed_url, :website_url, :scraper_adapter
     sanitizes_hash_attributes :scrape_settings, :custom_headers, :metadata
@@ -61,9 +61,28 @@ module SourceMonitor
         active.where(arel_table[:next_fetch_at].eq(nil).or(arel_table[:next_fetch_at].lteq(reference_time)))
       end
 
+      def scrape_candidates(threshold: SourceMonitor.config.scraping.scrape_recommendation_threshold)
+        threshold_value = threshold.to_i
+        return none if threshold_value <= 0
+
+        active
+          .where(scraping_enabled: false)
+          .where(
+            "#{table_name}.id IN (
+              SELECT i.source_id
+              FROM #{Item.table_name} i
+              INNER JOIN #{ItemContent.table_name} ic ON ic.item_id = i.id
+              WHERE ic.feed_word_count IS NOT NULL
+              GROUP BY i.source_id
+              HAVING AVG(ic.feed_word_count) < ?
+            )", threshold_value
+          )
+      end
+
       def ransackable_attributes(_auth_object = nil)
         %w[name feed_url website_url created_at fetch_interval_minutes items_count last_fetched_at
-           active health_status feed_format scraper_adapter]
+           active health_status feed_format scraper_adapter scraping_enabled
+           new_items_per_day avg_feed_words avg_scraped_words]
       end
 
       def ransackable_associations(_auth_object = nil)
@@ -71,6 +90,32 @@ module SourceMonitor
       end
     end
 
+    ransacker :new_items_per_day do
+      Arel.sql(
+        "(SELECT COUNT(*) / 30.0 FROM #{Item.table_name} i" \
+        " WHERE i.source_id = #{table_name}.id" \
+        " AND i.created_at >= NOW() - INTERVAL '30 days')"
+      )
+    end
+
+    ransacker :avg_feed_words do
+      Arel.sql(
+        "(SELECT AVG(ic.feed_word_count) FROM #{ItemContent.table_name} ic" \
+        " INNER JOIN #{Item.table_name} i ON i.id = ic.item_id" \
+        " WHERE i.source_id = #{table_name}.id" \
+        " AND ic.feed_word_count IS NOT NULL)"
+      )
+    end
+
+    ransacker :avg_scraped_words do
+      Arel.sql(
+        "(SELECT AVG(ic.scraped_word_count) FROM #{ItemContent.table_name} ic" \
+        " INNER JOIN #{Item.table_name} i ON i.id = ic.item_id" \
+        " WHERE i.source_id = #{table_name}.id" \
+        " AND ic.scraped_word_count IS NOT NULL)"
+      )
+    end
+
     def fetch_interval_minutes=(value)
       self[:fetch_interval_minutes] = value.presence && value.to_i
     end