source_monitor 0.10.2 → 0.11.0

data/app/views/source_monitor/sources/_row.html.erb

@@ -1,6 +1,7 @@
 <% rate_map = local_assigns[:item_activity_rates] || {} %>
 <% avg_feed_words_map = local_assigns[:avg_feed_word_counts] || {} %>
 <% avg_scraped_words_map = local_assigns[:avg_scraped_word_counts] || {} %>
+<% scrape_candidates = local_assigns[:scrape_candidate_ids] || Set.new %>
 <% activity_rate = rate_map.fetch(source.id, 0.0) %>
 <% health_status_override = local_assigns[:health_status_override] %>
 <% health_status = if !source.active?
@@ -24,22 +25,48 @@
 <% delete_path = delete_query.present? ? source_monitor.source_path(source, q: delete_query) : source_monitor.source_path(source) %>
 
 <tr id="<%= dom_id(source, :row) %>" class="hover:bg-slate-50">
+  <td class="w-10 px-3 py-4">
+    <% if scrape_candidates.include?(source.id) %>
+      <input type="checkbox"
+             name="bulk_scrape_enablement[source_ids][]"
+             value="<%= source.id %>"
+             data-select-all-target="item"
+             data-action="select-all#toggleItem"
+             class="rounded border-slate-300 text-violet-600 focus:ring-violet-500"
+             aria-label="Select <%= source.name %>">
+    <% end %>
+  </td>
   <td class="px-6 py-4">
     <div class="flex items-center gap-3">
       <%= source_favicon_tag(source, size: 24) %>
       <div>
-        <div class="font-medium text-slate-900">
+        <div class="flex items-center gap-2 font-medium text-slate-900">
           <%= link_to source.name,
                 source_monitor.source_path(source),
                 class: "text-slate-900 hover:text-blue-600 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2",
                 data: { turbo_frame: "_top" } %>
+          <% if source.last_error.to_s.match?(/\bblocked\b/i) %>
+            <% blocked_detail = source.last_error.to_s.match(/blocked by (\w+)/i) %>
+            <span class="inline-flex items-center rounded-full bg-rose-100 px-2 py-0.5 text-[10px] font-semibold text-rose-700"
+                  title="<%= blocked_detail ? "#{blocked_detail[1].capitalize} Blocked" : "Blocked" %>"
+                  data-testid="source-blocked-badge">
+              Blocked
+            </span>
+          <% end %>
         </div>
         <div class="text-xs text-slate-500 truncate max-w-xs"><%= external_link_to source.feed_url, source.feed_url, class: "text-slate-500 hover:text-blue-500" %></div>
+        <% if scrape_candidates.include?(source.id) %>
+          <span class="mt-1 inline-flex items-center rounded-full bg-violet-100 px-2 py-0.5 text-[10px] font-semibold text-violet-700"
+                title="Low feed word count — consider enabling scraping"
+                data-testid="scrape-recommendation-badge">
+            Scrape Recommended
+          </span>
+        <% end %>
       </div>
     </div>
   </td>
   <td class="px-6 py-4">
-    <div class="flex flex-col gap-2 text-xs">
+    <div class="flex flex-col gap-1 text-xs">
       <% if source.active? %>
         <%= render "source_monitor/sources/health_status_badge",
               source: source,
@@ -50,6 +77,10 @@
       <% if source.rolling_success_rate.present? %>
         <span class="text-[11px] text-slate-500">Success Rate: <%= number_to_percentage(source.rolling_success_rate * 100, precision: 0) %></span>
       <% end %>
+    </div>
+  </td>
+  <td class="px-6 py-4">
+    <div class="flex flex-col gap-1 text-xs">
       <span class="inline-flex items-center rounded-full px-3 py-1 font-semibold <%= fetch_status[:classes] %>">
         <%= loading_spinner_svg(css_class: "mr-1 h-3.5 w-3.5 animate-spin text-blue-500") if fetch_status[:show_spinner] %>
         <%= fetch_status[:label] %>
@@ -57,12 +88,9 @@
           <span class="ml-2 font-normal text-[10px] text-slate-500">(since <%= source.last_fetch_started_at.strftime("%H:%M:%S") %>)</span>
         <% end %>
       </span>
+      <span class="text-[11px] text-slate-500">(<%= number_with_precision(1440.0 / source.fetch_interval_minutes, precision: 1) %>x / day)</span>
     </div>
   </td>
-  <td class="px-6 py-4 text-sm">
-    <%= "#{source.fetch_interval_minutes} min" %>
-    <span class="text-xs text-slate-500">(~<%= number_with_precision(source.fetch_interval_minutes / 60.0, precision: 2) %> h)</span>
-  </td>
   <td class="px-6 py-4 text-sm"><%= source.items_count %></td>
   <td class="px-6 py-4 text-sm">
     <%= number_with_precision(activity_rate, precision: 2) %>

data/app/views/source_monitor/sources/index.html.erb

@@ -12,6 +12,13 @@
     </div>
   </div>
 
+  <%= render "source_monitor/sources/import_history_panel", import_histories: @recent_import_histories %>
+
+  <%= render "source_monitor/sources/fetch_interval_heatmap",
+        fetch_interval_distribution: @fetch_interval_distribution,
+        selected_bucket: @selected_fetch_interval_bucket,
+        search_params: @search_params %>
+
   <%= search_form_for @q, url: source_monitor.sources_path, method: :get, html: { class: "flex flex-wrap items-end gap-3", data: { turbo_frame: "source_monitor_sources_table" } } do |form| %>
     <div class="flex-1 min-w-[12rem]">
       <%= form.label @search_field, "Search sources", class: "sr-only" %>
@@ -27,7 +34,7 @@
       </div>
       <div>
         <%= form.label :health_status_eq, "Health", class: "block text-xs font-medium text-slate-500 mb-1" %>
-        <%= form.select :health_status_eq, options_for_select([["All Health", ""], ["Healthy", "healthy"], ["Warning", "warning"], ["Declining", "declining"], ["Critical", "critical"]], @search_params["health_status_eq"].to_s), {}, class: "rounded-md border border-slate-200 bg-white px-2 py-2 text-sm text-slate-700 focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500", onchange: "this.form.requestSubmit()" %>
+        <%= form.select :health_status_eq, options_for_select([["All Health", ""], ["Working", "working"], ["Declining", "declining"], ["Improving", "improving"], ["Failing", "failing"]], @search_params["health_status_eq"].to_s), {}, class: "rounded-md border border-slate-200 bg-white px-2 py-2 text-sm text-slate-700 focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500", onchange: "this.form.requestSubmit()" %>
       </div>
       <div>
         <%= form.label :feed_format_eq, "Format", class: "block text-xs font-medium text-slate-500 mb-1" %>
@@ -38,19 +45,16 @@
         <%= form.label :scraper_adapter_eq, "Adapter", class: "block text-xs font-medium text-slate-500 mb-1" %>
         <%= form.select :scraper_adapter_eq, options_for_select([["All Adapters", ""]] + adapter_options.map { |a| [a.titleize, a] }, @search_params["scraper_adapter_eq"].to_s), {}, class: "rounded-md border border-slate-200 bg-white px-2 py-2 text-sm text-slate-700 focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500", onchange: "this.form.requestSubmit()" %>
       </div>
+      <div>
+        <%= form.label :scraping_enabled_eq, "Scrape", class: "block text-xs font-medium text-slate-500 mb-1" %>
+        <%= form.select :scraping_enabled_eq, options_for_select([["All Sources", ""], ["Scraping Enabled", "true"], ["Scraping Disabled", "false"], ["Recommendations", "recommend"]], @search_params["scraping_enabled_eq"].to_s), {}, class: "rounded-md border border-slate-200 bg-white px-2 py-2 text-sm text-slate-700 focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500", onchange: "this.form.requestSubmit()" %>
+      </div>
     </div>
   <% end %>
 
-  <%= render "source_monitor/sources/import_history_panel", import_histories: @recent_import_histories %>
-
-  <%= render "source_monitor/sources/fetch_interval_heatmap",
-        fetch_interval_distribution: @fetch_interval_distribution,
-        selected_bucket: @selected_fetch_interval_bucket,
-        search_params: @search_params %>
-
   <div class="overflow-hidden rounded-lg border border-slate-200 bg-white shadow-sm">
     <%= turbo_frame_tag "source_monitor_sources_table" do %>
-      <% dropdown_filter_keys = %w[active_eq health_status_eq feed_format_eq scraper_adapter_eq] %>
+      <% dropdown_filter_keys = %w[active_eq health_status_eq feed_format_eq scraper_adapter_eq scraping_enabled_eq avg_feed_words_lt] %>
       <% active_dropdown_filters = dropdown_filter_keys.select { |k| @search_params[k].present? } %>
       <% has_any_filter = @search_term.present? || @fetch_interval_filter.present? || active_dropdown_filters.any? %>
       <% if has_any_filter %>
@@ -87,7 +91,9 @@
                 "active_eq" => @search_params["active_eq"] == "true" ? "Status: Active" : "Status: Paused",
                 "health_status_eq" => "Health: #{@search_params['health_status_eq']&.titleize}",
                 "feed_format_eq" => "Format: #{@search_params['feed_format_eq']&.upcase}",
-                "scraper_adapter_eq" => "Adapter: #{@search_params['scraper_adapter_eq']&.titleize}"
+                "scraper_adapter_eq" => "Adapter: #{@search_params['scraper_adapter_eq']&.titleize}",
+                "scraping_enabled_eq" => @search_params["scraping_enabled_eq"] == "true" ? "Scraping: Enabled" : "Scraping: Disabled",
+                "avg_feed_words_lt" => "Avg Feed Words: < #{@search_params['avg_feed_words_lt']}"
               } %>
               <% active_dropdown_filters.each do |filter_key| %>
                 <% clear_query = @search_params.dup %>
@@ -107,131 +113,177 @@
           <% end %>
         </div>
       <% end %>
-      <table class="min-w-full divide-y divide-slate-200 text-left text-sm">
-        <thead class="bg-slate-50 text-xs font-semibold uppercase tracking-wide text-slate-500">
-          <tr>
-            <th scope="col"
-                class="px-6 py-3"
-                data-sort-column="name"
-                aria-sort="<%= table_sort_aria(@q, :name) %>">
-              <span class="inline-flex items-center gap-1">
-                <%= table_sort_link(
-                      @q,
-                      :name,
-                      "Source",
-                      frame: "source_monitor_sources_table",
-                      default_order: :asc,
-                      secondary: ["created_at desc"],
-                      html_options: {
-                        class: "inline-flex items-center gap-1 text-xs font-semibold uppercase tracking-wide text-slate-600 hover:text-slate-900 focus:outline-none"
-                      }
-                    ) %>
-                <span class="text-[11px] text-slate-400" aria-hidden="true"><%= table_sort_arrow(@q, :name) %></span>
-              </span>
-            </th>
-            <th scope="col" class="px-6 py-3">Status</th>
-            <th scope="col"
-                class="px-6 py-3"
-                data-sort-column="fetch_interval_minutes"
-                aria-sort="<%= table_sort_aria(@q, :fetch_interval_minutes) %>">
-              <span class="inline-flex items-center gap-1">
-                <%= table_sort_link(
-                      @q,
-                      :fetch_interval_minutes,
-                      "Fetch Interval",
-                      frame: "source_monitor_sources_table",
-                      default_order: :asc,
-                      secondary: ["created_at desc"],
-                      html_options: {
-                        class: "inline-flex items-center gap-1 text-xs font-semibold uppercase tracking-wide text-slate-600 hover:text-slate-900 focus:outline-none"
-                      }
-                    ) %>
-                <span class="text-[11px] text-slate-400" aria-hidden="true"><%= table_sort_arrow(@q, :fetch_interval_minutes) %></span>
-              </span>
-            </th>
-            <th scope="col"
-                class="px-6 py-3"
-                data-sort-column="items_count"
-                aria-sort="<%= table_sort_aria(@q, :items_count) %>">
-              <span class="inline-flex items-center gap-1">
-                <%= table_sort_link(
-                      @q,
-                      :items_count,
-                      "Items",
-                      frame: "source_monitor_sources_table",
-                      default_order: :desc,
-                      secondary: ["created_at desc"],
-                      html_options: {
-                        class: "inline-flex items-center gap-1 text-xs font-semibold uppercase tracking-wide text-slate-600 hover:text-slate-900 focus:outline-none"
-                      }
-                    ) %>
-                <span class="text-[11px] text-slate-400" aria-hidden="true"><%= table_sort_arrow(@q, :items_count) %></span>
-              </span>
-            </th>
-            <th scope="col" class="px-6 py-3">New Items / Day</th>
-            <th scope="col" class="px-6 py-3">Avg Feed Words</th>
-            <th scope="col" class="px-6 py-3">Avg Scraped Words</th>
-            <th scope="col"
-                class="px-6 py-3"
-                data-sort-column="last_fetched_at"
-                aria-sort="<%= table_sort_aria(@q, :last_fetched_at) %>">
-              <span class="inline-flex items-center gap-1">
-                <%= table_sort_link(
-                      @q,
-                      :last_fetched_at,
-                      "Last Fetch",
-                      frame: "source_monitor_sources_table",
-                      default_order: :desc,
-                      secondary: ["created_at desc"],
-                      html_options: {
-                        class: "inline-flex items-center gap-1 text-xs font-semibold uppercase tracking-wide text-slate-600 hover:text-slate-900 focus:outline-none"
-                      }
-                    ) %>
-                <span class="text-[11px] text-slate-400" aria-hidden="true"><%= table_sort_arrow(@q, :last_fetched_at) %></span>
-              </span>
-            </th>
-            <th scope="col" class="px-6 py-3"></th>
-          </tr>
-        </thead>
-        <tbody id="source_monitor_sources_table_body" class="divide-y divide-slate-100 text-slate-700">
-          <%= render partial: "source_monitor/sources/row",
-                collection: @sources,
-                as: :source,
-                locals: {
-                  item_activity_rates: @item_activity_rates,
-                  avg_feed_word_counts: @avg_feed_word_counts,
-                  avg_scraped_word_counts: @avg_scraped_word_counts,
-                  search_params: @search_params
-                } %>
-
-          <%= render("source_monitor/sources/empty_state_row") if @sources.blank? %>
-        </tbody>
-      </table>
-      <div class="flex flex-col items-center gap-3 border-t border-slate-200 px-6 py-4 sm:flex-row sm:justify-between">
-        <div class="text-xs text-slate-500">
-          Page <%= @page %>
-        </div>
-        <div class="flex gap-2">
-          <% prev_params = { page: @page - 1 } %>
-          <% prev_params[:q] = @search_params if @search_params.present? %>
-          <% prev_params[:per_page] = params[:per_page] if params[:per_page].present? %>
-          <% next_params = { page: @page + 1 } %>
-          <% next_params[:q] = @search_params if @search_params.present? %>
-          <% next_params[:per_page] = params[:per_page] if params[:per_page].present? %>
+      <%= form_with url: source_monitor.bulk_scrape_enablements_path, data: { controller: "select-all" } do |form| %>
+        <table class="min-w-full divide-y divide-slate-200 text-left text-sm">
+          <thead class="bg-slate-50 text-xs font-semibold uppercase tracking-wide text-slate-500">
+            <tr>
+              <th scope="col" class="w-10 px-3 py-3">
+                <% if @scrape_candidate_ids.any? %>
+                  <input type="checkbox"
+                         data-select-all-target="master"
+                         data-action="select-all#toggleAll"
+                         class="rounded border-slate-300 text-violet-600 focus:ring-violet-500"
+                         aria-label="Select all sources">
+                <% end %>
+              </th>
+              <th scope="col"
+                  class="px-6 py-3"
+                  data-sort-column="name"
+                  aria-sort="<%= table_sort_aria(@q, :name) %>">
+                <span class="inline-flex items-center gap-1">
+                  <%= table_sort_link(
+                        @q,
+                        :name,
+                        "Source",
+                        frame: "source_monitor_sources_table",
+                        default_order: :asc,
+                        secondary: ["created_at desc"],
+                        html_options: {
+                          class: "inline-flex items-center gap-1 text-xs font-semibold uppercase tracking-wide text-slate-600 hover:text-slate-900 focus:outline-none"
+                        }
+                      ) %>
+                  <span class="text-[11px] text-slate-400" aria-hidden="true"><%= table_sort_arrow(@q, :name) %></span>
+                </span>
+              </th>
+              <th scope="col" class="px-6 py-3">Health</th>
+              <th scope="col" class="px-6 py-3">Fetch Status</th>
+              <th scope="col"
+                  class="px-6 py-3"
+                  data-sort-column="items_count"
+                  aria-sort="<%= table_sort_aria(@q, :items_count) %>">
+                <span class="inline-flex items-center gap-1">
+                  <%= table_sort_link(
+                        @q,
+                        :items_count,
+                        "Items",
+                        frame: "source_monitor_sources_table",
+                        default_order: :desc,
+                        secondary: ["created_at desc"],
+                        html_options: {
+                          class: "inline-flex items-center gap-1 text-xs font-semibold uppercase tracking-wide text-slate-600 hover:text-slate-900 focus:outline-none"
+                        }
+                      ) %>
+                  <span class="text-[11px] text-slate-400" aria-hidden="true"><%= table_sort_arrow(@q, :items_count) %></span>
+                </span>
+              </th>
+              <th scope="col"
+                  class="px-6 py-3"
+                  data-sort-column="new_items_per_day"
+                  aria-sort="<%= table_sort_aria(@q, :new_items_per_day) %>">
+                <span class="inline-flex items-center gap-1">
+                  <%= table_sort_link(
+                        @q,
+                        :new_items_per_day,
+                        "New Items / Day",
+                        frame: "source_monitor_sources_table",
+                        default_order: :desc,
+                        secondary: ["created_at desc"],
+                        html_options: {
+                          class: "inline-flex items-center gap-1 text-xs font-semibold uppercase tracking-wide text-slate-600 hover:text-slate-900 focus:outline-none"
+                        }
+                      ) %>
+                  <span class="text-[11px] text-slate-400" aria-hidden="true"><%= table_sort_arrow(@q, :new_items_per_day) %></span>
+                </span>
+              </th>
+              <th scope="col"
+                  class="px-6 py-3"
+                  data-sort-column="avg_feed_words"
+                  aria-sort="<%= table_sort_aria(@q, :avg_feed_words) %>">
+                <span class="inline-flex items-center gap-1">
+                  <%= table_sort_link(
+                        @q,
+                        :avg_feed_words,
+                        "Avg Feed Words",
+                        frame: "source_monitor_sources_table",
+                        default_order: :desc,
+                        secondary: ["created_at desc"],
+                        html_options: {
+                          class: "inline-flex items-center gap-1 text-xs font-semibold uppercase tracking-wide text-slate-600 hover:text-slate-900 focus:outline-none"
+                        }
+                      ) %>
+                  <span class="text-[11px] text-slate-400" aria-hidden="true"><%= table_sort_arrow(@q, :avg_feed_words) %></span>
+                </span>
+              </th>
+              <th scope="col"
+                  class="px-6 py-3"
+                  data-sort-column="avg_scraped_words"
+                  aria-sort="<%= table_sort_aria(@q, :avg_scraped_words) %>">
+                <span class="inline-flex items-center gap-1">
+                  <%= table_sort_link(
+                        @q,
+                        :avg_scraped_words,
+                        "Avg Scraped Words",
+                        frame: "source_monitor_sources_table",
+                        default_order: :desc,
+                        secondary: ["created_at desc"],
+                        html_options: {
+                          class: "inline-flex items-center gap-1 text-xs font-semibold uppercase tracking-wide text-slate-600 hover:text-slate-900 focus:outline-none"
+                        }
+                      ) %>
+                  <span class="text-[11px] text-slate-400" aria-hidden="true"><%= table_sort_arrow(@q, :avg_scraped_words) %></span>
+                </span>
+              </th>
+              <th scope="col"
+                  class="px-6 py-3"
+                  data-sort-column="last_fetched_at"
+                  aria-sort="<%= table_sort_aria(@q, :last_fetched_at) %>">
+                <span class="inline-flex items-center gap-1">
+                  <%= table_sort_link(
+                        @q,
+                        :last_fetched_at,
+                        "Last Fetch",
+                        frame: "source_monitor_sources_table",
+                        default_order: :desc,
+                        secondary: ["created_at desc"],
+                        html_options: {
+                          class: "inline-flex items-center gap-1 text-xs font-semibold uppercase tracking-wide text-slate-600 hover:text-slate-900 focus:outline-none"
+                        }
+                      ) %>
+                  <span class="text-[11px] text-slate-400" aria-hidden="true"><%= table_sort_arrow(@q, :last_fetched_at) %></span>
+                </span>
+              </th>
+              <th scope="col" class="px-6 py-3"></th>
+            </tr>
+          </thead>
+          <tbody id="source_monitor_sources_table_body" class="divide-y divide-slate-100 text-slate-700">
+            <%= render partial: "source_monitor/sources/row",
+                  collection: @sources,
+                  as: :source,
+                  locals: {
+                    item_activity_rates: @item_activity_rates,
+                    avg_feed_word_counts: @avg_feed_word_counts,
+                    avg_scraped_word_counts: @avg_scraped_word_counts,
+                    search_params: @search_params,
+                    scrape_candidate_ids: @scrape_candidate_ids
+                  } %>
 
-          <% if @has_previous_page %>
-            <%= link_to "Previous", source_monitor.sources_path(prev_params), class: "inline-flex items-center rounded-md border border-slate-300 px-3 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50", data: { turbo_frame: "source_monitor_sources_table" } %>
-          <% else %>
-            <span class="inline-flex items-center rounded-md border border-slate-200 px-3 py-2 text-sm font-medium text-slate-300">Previous</span>
-          <% end %>
+            <%= render("source_monitor/sources/empty_state_row") if @sources.blank? %>
+          </tbody>
+        </table>
 
-          <% if @has_next_page %>
-            <%= link_to "Next", source_monitor.sources_path(next_params), class: "inline-flex items-center rounded-md border border-slate-300 px-3 py-2 text-sm font-medium text-slate-700 hover:bg-slate-50", data: { turbo_frame: "source_monitor_sources_table" } %>
-          <% else %>
-            <span class="inline-flex items-center rounded-md border border-slate-200 px-3 py-2 text-sm font-medium text-slate-300">Next</span>
-          <% end %>
+        <div data-select-all-target="actionBar" class="hidden sticky bottom-0 border-t border-slate-200 bg-white px-4 py-3 shadow-md">
+          <div class="flex items-center justify-between">
+            <span class="text-sm text-slate-700">
+              <span data-select-all-target="count">0</span> source(s) selected
+            </span>
+            <button type="button"
+                    data-action="modal#open"
+                    class="inline-flex items-center rounded-md bg-violet-600 px-4 py-2 text-sm font-semibold text-white shadow hover:bg-violet-500">
+              Enable Scraping
+            </button>
+          </div>
         </div>
-      </div>
+
+        <%= render "source_monitor/sources/bulk_scrape_enable_modal" %>
+      <% end %>
+      <% extra_params = {} %>
+      <% extra_params[:q] = @search_params if @search_params.present? %>
+      <% extra_params[:per_page] = params[:per_page] if params[:per_page].present? %>
+      <%= render "source_monitor/shared/pagination",
+            paginator_result: @paginator,
+            base_path: source_monitor.sources_path,
+            extra_params: extra_params,
+            turbo_frame: "source_monitor_sources_table" %>
     <% end %>
   </div>
 </div>

data/config/brakeman.ignore

@@ -10,8 +10,18 @@
       "line": 77,
       "code": "OpenSSL::SSL::SSLContext.new.verify_mode = OpenSSL::SSL::VERIFY_NONE",
       "note": "Intentional: AIA resolver must connect without verification to fetch the leaf certificate from servers with broken certificate chains. This is the core purpose of the module -- it only uses VERIFY_NONE to read the cert, never to transmit data."
+    },
+    {
+      "warning_type": "Mass Assignment",
+      "warning_code": 70,
+      "fingerprint": "b4702d88859ff3c4f9e954eac9335d003834c1d37797c54587a0dc9b3b76bf8b",
+      "check_name": "MassAssignment",
+      "message": "Specify exact keys allowed for mass assignment instead of using `permit!` which allows any keys",
+      "file": "app/controllers/source_monitor/dashboard_controller.rb",
+      "line": 33,
+      "note": "Safe: schedule_pages params are only used as page numbers for Turbo Frame pagination of fetch schedule buckets. No model attributes are assigned from these values."
     }
   ],
-  "updated": "2026-02-17",
+  "updated": "2026-03-13",
   "brakeman_version": "8.0.2"
 }

data/config/routes.rb

@@ -7,6 +7,9 @@ SourceMonitor::Engine.routes.draw do
   resources :logs, only: :index
   resources :fetch_logs, only: :show
   resources :scrape_logs, only: :show
+  resources :import_histories, only: [] do
+    resource :dismissal, only: :create, controller: "import_history_dismissals"
+  end
   resources :import_sessions, path: "import_opml", only: %i[new create show update destroy] do
     member do
       get "steps/:step", action: :show, as: :step
@@ -16,6 +19,7 @@ SourceMonitor::Engine.routes.draw do
   resources :items, only: %i[index show] do
     post :scrape, on: :member
   end
+  resources :bulk_scrape_enablements, only: :create
   resources :sources do
     resource :fetch, only: :create, controller: "source_fetches"
     resource :retry, only: :create, controller: "source_retries"
@@ -23,5 +27,6 @@ SourceMonitor::Engine.routes.draw do
     resource :health_check, only: :create, controller: "source_health_checks"
     resource :health_reset, only: :create, controller: "source_health_resets"
     resource :favicon_fetch, only: :create, controller: "source_favicon_fetches"
+    resource :scrape_test, only: :create, controller: "source_scrape_tests"
   end
 end

data/db/migrate/20260305120000_add_dismissed_at_to_import_histories.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddDismissedAtToImportHistories < ActiveRecord::Migration[8.1]
+  def change
+    add_column :"#{SourceMonitor.table_name_prefix}import_histories", :dismissed_at, :datetime
+  end
+end

data/db/migrate/20260306233004_add_error_category_to_fetch_logs.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class AddErrorCategoryToFetchLogs < ActiveRecord::Migration[8.0]
+  def change
+    add_column :sourcemon_fetch_logs, :error_category, :string
+    add_index :sourcemon_fetch_logs, :error_category
+  end
+end

data/db/migrate/20260307120000_add_consecutive_fetch_failures_to_sources.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class AddConsecutiveFetchFailuresToSources < ActiveRecord::Migration[8.0]
+  def change
+    add_column :sourcemon_sources, :consecutive_fetch_failures, :integer, default: 0, null: false
+
+    add_index :sourcemon_sources, :consecutive_fetch_failures,
+              where: "consecutive_fetch_failures > 0",
+              name: "index_sources_on_consecutive_failures"
+  end
+end

data/db/migrate/20260312120000_simplify_health_status_values.rb

@@ -0,0 +1,20 @@
+class SimplifyHealthStatusValues < ActiveRecord::Migration[8.0]
+  def up
+    execute <<~SQL
+      UPDATE sourcemon_sources SET health_status = 'working' WHERE health_status IN ('healthy', 'auto_paused', 'unknown')
+    SQL
+    execute <<~SQL
+      UPDATE sourcemon_sources SET health_status = 'failing' WHERE health_status IN ('warning', 'critical')
+    SQL
+    # 'declining' and 'improving' remain unchanged
+  end
+
+  def down
+    execute <<~SQL
+      UPDATE sourcemon_sources SET health_status = 'healthy' WHERE health_status = 'working'
+    SQL
+    execute <<~SQL
+      UPDATE sourcemon_sources SET health_status = 'critical' WHERE health_status = 'failing'
+    SQL
+  end
+end

data/docs/configuration.md

@@ -86,6 +86,14 @@ config.scrapers.register(:custom, "MyApp::Scrapers::Premium" )
 
 Adapters receive merged settings (`default -> source -> invocation`), and must return a `SourceMonitor::Scrapers::Result` object. Use `config.scrapers.unregister(:custom)` to remove overrides.
 
+## Scraping Settings
+
+`config.scraping` controls scrape concurrency and recommendations.
+
+- `max_in_flight_per_source` – max concurrent scrape jobs per source (`nil` = unlimited, default `nil`)
+- `max_bulk_batch_size` – max items per bulk scrape enqueue (default `100`)
+- `scrape_recommendation_threshold` – minimum average feed word count below which a source is recommended for scraping (default `200`)
+
 ## Events & Item Processors
 
 Respond to lifecycle events without monkey patching:
@@ -147,7 +155,7 @@ Handlers can be symbols (invoked on the controller) or callables. Return `false`
 `config.health` tunes automatic pause/resume heuristics.
 
 - `window_size` – number of fetch attempts to evaluate (default `20`)
-- `healthy_threshold` / `warning_threshold` – ratios that drive UI badges
+- `healthy_threshold` – ratio that drives the "working" badge
 - `auto_pause_threshold` / `auto_resume_threshold` – percentages that trigger automatic toggling
 - `auto_pause_cooldown_minutes` – grace period before re-enabling a source
 

data/docs/troubleshooting.md

@@ -109,6 +109,15 @@ This guide lists common issues you might encounter while installing, upgrading,
 - Fix by running `npm install` followed by `npm run build` inside the engine root so that `app/assets/builds/source_monitor/application.css` and `application.js` exist. The Rake task `app:source_monitor:assets:build` wraps the same scripts for CI usage.
 - When the UI is still unstyled, confirm the dummy app can read the namespaced asset directories noted in `.ai/engine-asset-configuration.md:32-44` and restart `bin/dev` so the CSS/JS watchers reconnect.
 
+## 14. Feed Returns Cloudflare Challenge Page
+
+- **Symptoms:** Fetch logs show HTML containing "Checking your browser" or "Just a moment..." instead of feed XML. The source may display a "Blocked" badge.
+- **How it works:** SourceMonitor automatically detects Cloudflare challenge responses and attempts cookie replay with UA rotation on subsequent fetches. No configuration is needed -- this is enabled by default.
+- **If fetches remain blocked:** Some sites employ aggressive bot mitigation that cannot be bypassed with cookie replay alone. In these cases:
+  - Try setting a custom `user_agent` on the source's custom headers to match a specific browser.
+  - Consider whether the site offers an alternative feed URL that is not behind Cloudflare.
+  - Check the fetch log's `error_category` field -- a value of `blocked` confirms Cloudflare detection triggered.
+
 ## Still Stuck?
 
 Collect the following and open an issue or start a discussion:

data/docs/upgrade.md

@@ -46,6 +46,37 @@ If a removed option raises an error (`SourceMonitor::DeprecatedOptionError`), yo
 
 ## Version-Specific Notes
 
+### Upgrading to 0.11.0
+
+**What changed:**
+- Health status values simplified from 7 (`healthy`, `warning`, `critical`, `declining`, `improving`, `auto_paused`, `unknown`) to 4 (`working`, `declining`, `improving`, `failing`).
+- `warning_threshold` configuration setting removed entirely.
+- Auto-pause is now tracked as operational state (via `auto_paused_at`/`auto_paused_until` columns) rather than as a health status value. Sources can be both "failing AND auto-paused".
+- New `determine_status` decision tree uses rate thresholds and streak detection.
+- Dashboard and sources index updated with new badge colors: working (green), declining (yellow), improving (sky), failing (rose).
+- New `consecutive_fetch_failures` column on sources for streak-based health detection.
+- New `error_category` column on fetch logs for classifying failure types (e.g., timeout, DNS, blocked).
+- New `config.scraping.scrape_recommendation_threshold` setting (default 200) controls the word-count threshold for scrape recommendations on the dashboard.
+- Dashboard pagination for sources and items lists.
+- Automatic Cloudflare bypass via cookie replay and UA rotation (no configuration needed).
+- Smart scrape recommendations widget on the dashboard highlights sources that may benefit from scraping.
+
+**Upgrade steps:**
+```bash
+bundle update source_monitor
+bin/rails source_monitor:upgrade
+bin/rails db:migrate
+```
+
+**Notes:**
+- **Action required:** If your initializer sets `config.health.warning_threshold`, remove that line. The setting no longer exists and will raise an error.
+- The data migration automatically remaps existing health_status values: `healthy`/`auto_paused`/`unknown` become `working`, `warning`/`critical` become `failing`. This is reversible.
+- If your host app queries `health_status` directly (e.g., `Source.where(health_status: "healthy")`), update those queries to use the new values.
+- The `auto_paused?` method and `auto_paused_at`/`auto_paused_until` columns still exist and work the same way. Only the `health_status` column values changed.
+- `healthy_threshold` config setting still exists but now drives the `working` status (renamed from `healthy`).
+- The `consecutive_fetch_failures` and `error_category` columns are added via migrations and require no configuration.
+- Cloudflare bypass is automatic -- sources that return Cloudflare challenge pages will show a "Blocked" badge and the engine retries with cookie replay and UA rotation.
+
 ### Upgrading to 0.10.0 (from 0.9.x)
 
 **What changed:**