source_monitor 0.1.1

data/app/controllers/source_monitor/source_health_resets_controller.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class SourceHealthResetsController < ApplicationController
+    include SourceMonitor::SourceTurboResponses
+
+    before_action :set_source
+
+    def create
+      SourceMonitor::Health::SourceHealthReset.call(source: @source)
+      SourceMonitor::Realtime.broadcast_source(@source)
+
+      render_fetch_enqueue_response(
+        "Health state reset",
+        toast_level: :success
+      )
+    rescue StandardError => error
+      handle_fetch_failure(error, prefix: "Health reset")
+    end
+
+    private
+
+    def set_source
+      @source = Source.find(params[:source_id])
+    end
+  end
+end

data/app/controllers/source_monitor/source_retries_controller.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  class SourceRetriesController < ApplicationController
+    include SourceMonitor::SourceTurboResponses
+
+    before_action :set_source
+
+    def create
+      SourceMonitor::Fetching::FetchRunner.enqueue(@source.id, force: true)
+      render_fetch_enqueue_response("Retry has been forced and will run shortly.")
+    rescue StandardError => error
+      handle_fetch_failure(error)
+    end
+
+    private
+
+    def set_source
+      @source = Source.find(params[:source_id])
+    end
+  end
+end

data/app/controllers/source_monitor/source_turbo_responses.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module SourceTurboResponses
+    extend ActiveSupport::Concern
+
+    included do
+      include ActionView::RecordIdentifier
+    end
+
+    private
+
+    def render_fetch_enqueue_response(message, toast_level: :info, health_status_override: nil)
+      @source.reload
+      respond_to do |format|
+        format.turbo_stream do
+          responder = SourceMonitor::TurboStreams::StreamResponder.new
+
+          responder.replace_details(
+            @source,
+            partial: "source_monitor/sources/details_wrapper",
+            locals: { source: @source, health_status_override: health_status_override }
+          )
+
+          responder.replace_row(
+            @source,
+            partial: "source_monitor/sources/row",
+            locals: {
+              source: @source,
+              item_activity_rates: { @source.id => SourceMonitor::Analytics::SourceActivityRates.rate_for(@source) },
+              health_status_override: health_status_override
+            }
+          )
+
+          responder.toast(message:, level: toast_level, delay_ms: toast_delay_for(toast_level))
+
+          render turbo_stream: responder.render(view_context)
+        end
+
+        format.html do
+          redirect_to source_monitor.source_path(@source), notice: message
+        end
+      end
+    end
+
+    def handle_fetch_failure(error, prefix: "Fetch")
+      error_message = "#{prefix} could not be enqueued: #{error.message}"
+
+      respond_to do |format|
+        format.turbo_stream do
+          responder = SourceMonitor::TurboStreams::StreamResponder.new
+          responder.toast(message: error_message, level: :error, delay_ms: toast_delay_for(:error))
+
+          render turbo_stream: responder.render(view_context), status: :unprocessable_entity
+        end
+
+        format.html do
+          redirect_to source_monitor.source_path(@source), alert: error_message
+        end
+      end
+    end
+
+    def respond_to_bulk_scrape(result)
+      @source.reload
+      @bulk_scrape_selection = result.selection
+      payload = bulk_scrape_flash_payload(result)
+      status = result.error? ? :unprocessable_entity : :ok
+
+      respond_to do |format|
+        format.turbo_stream do
+          responder = SourceMonitor::TurboStreams::StreamResponder.new
+
+          responder.replace_details(
+            @source,
+            partial: "source_monitor/sources/details_wrapper",
+            locals: { source: @source }
+          )
+
+          responder.replace_row(
+            @source,
+            partial: "source_monitor/sources/row",
+            locals: {
+              source: @source,
+              item_activity_rates: { @source.id => SourceMonitor::Analytics::SourceActivityRates.rate_for(@source) }
+            }
+          )
+
+          if payload[:message].present?
+            responder.toast(
+              message: payload[:message],
+              level: payload[:level],
+              delay_ms: toast_delay_for(payload[:level])
+            )
+          end
+
+          render turbo_stream: responder.render(view_context), status: status
+        end
+
+        format.html do
+          if payload[:message].present?
+            redirect_to source_monitor.source_path(@source), flash: { payload[:flash_key] => payload[:message] }
+          else
+            redirect_to source_monitor.source_path(@source)
+          end
+        end
+      end
+    end
+
+    def bulk_scrape_flash_payload(result)
+      pluralizer = ->(count, word) { view_context.pluralize(count, word) }
+      presenter = SourceMonitor::Scraping::BulkResultPresenter.new(result:, pluralizer:)
+      presenter.to_flash_payload
+    end
+  end
+end

data/app/controllers/source_monitor/sources_controller.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require "source_monitor/sources/turbo_stream_presenter"
+require "source_monitor/scraping/bulk_result_presenter"
+
+module SourceMonitor
+  class SourcesController < ApplicationController
+    include ActionView::RecordIdentifier
+    include SourceMonitor::SanitizesSearchParams
+
+    searchable_with scope: -> { Source.all }, default_sorts: [ "created_at desc" ]
+
+    ITEMS_PREVIEW_LIMIT = SourceMonitor::Scraping::BulkSourceScraper::DEFAULT_PREVIEW_LIMIT
+
+    before_action :set_source, only: %i[show edit update destroy]
+
+    SEARCH_FIELD = :name_or_feed_url_or_website_url_cont
+
+    def index
+      @search_params = sanitized_search_params
+      @q = build_search_query
+
+      @sources = @q.result
+
+      @search_term = @search_params[SEARCH_FIELD.to_s].to_s.strip
+      @search_field = SEARCH_FIELD
+
+      metrics = SourceMonitor::Analytics::SourcesIndexMetrics.new(
+        base_scope: Source.all,
+        result_scope: @sources,
+        search_params: @search_params
+      )
+
+      @fetch_interval_distribution = metrics.fetch_interval_distribution
+      @fetch_interval_filter = metrics.fetch_interval_filter
+      @selected_fetch_interval_bucket = metrics.selected_fetch_interval_bucket
+      @item_activity_rates = metrics.item_activity_rates
+    end
+
+    def show
+      @recent_fetch_logs = @source.fetch_logs.order(started_at: :desc).limit(5)
+      @recent_scrape_logs = @source.scrape_logs.order(started_at: :desc).limit(5)
+      @items = @source.items.recent.limit(ITEMS_PREVIEW_LIMIT)
+      @bulk_scrape_selection = :current
+    end
+
+    def new
+      @source = Source.new(default_attributes)
+    end
+
+    def create
+      @source = Source.new(source_params)
+
+      if @source.save
+        redirect_to source_monitor.source_path(@source), notice: "Source created successfully"
+      else
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def edit
+    end
+
+    def update
+      if @source.update(source_params)
+        redirect_to source_monitor.source_path(@source), notice: "Source updated successfully"
+      else
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    def destroy
+      search_params = sanitized_search_params
+      @source.destroy
+      message = "Source deleted"
+
+      respond_to do |format|
+        format.turbo_stream do
+          query = build_search_query(params: search_params)
+
+          metrics = SourceMonitor::Analytics::SourcesIndexMetrics.new(
+            base_scope: Source.all,
+            result_scope: query.result,
+            search_params:
+          )
+
+          redirect_location = safe_redirect_path(params[:redirect_to])
+
+          responder = SourceMonitor::TurboStreams::StreamResponder.new
+          presenter = SourceMonitor::Sources::TurboStreamPresenter.new(source: @source, responder:)
+
+          presenter.render_deletion(
+            metrics:,
+            query:,
+            search_params:,
+            redirect_location:
+          )
+
+          responder.toast(message:, level: :success)
+
+          render turbo_stream: responder.render(view_context)
+        end
+
+        format.html do
+          redirect_to source_monitor.sources_path, notice: message
+        end
+      end
+    end
+
+    def fetch
+      SourceMonitor::Fetching::FetchRunner.enqueue(@source.id)
+      render_fetch_enqueue_response("Fetch has been enqueued and will run shortly.")
+    rescue StandardError => error
+      handle_fetch_failure(error)
+    end
+
+    def retry
+      SourceMonitor::Fetching::FetchRunner.enqueue(@source.id, force: true)
+      render_fetch_enqueue_response("Retry has been forced and will run shortly.")
+    rescue StandardError => error
+      handle_fetch_failure(error)
+    end
+
+    private
+
+    def set_source
+      @source = Source.find(params[:id])
+    end
+
+    def default_attributes
+      {
+        active: true,
+        scraping_enabled: false,
+        auto_scrape: false,
+        requires_javascript: false,
+        feed_content_readability_enabled: false,
+        fetch_interval_minutes: 360,
+        adaptive_fetching_enabled: true,
+        scraper_adapter: "readability"
+      }
+    end
+
+    def source_params
+      permitted = params.require(:source).permit(
+        :name,
+        :feed_url,
+        :website_url,
+        :fetch_interval_minutes,
+        :active,
+        :auto_scrape,
+        :scraping_enabled,
+        :requires_javascript,
+        :feed_content_readability_enabled,
+        :scraper_adapter,
+        :items_retention_days,
+        :max_items,
+        :adaptive_fetching_enabled,
+        :health_auto_pause_threshold,
+        scrape_settings: [
+          :include_plain_text,
+          :timeout,
+          :javascript_enabled,
+          { selectors: %i[content title],
+            http: [],
+            readability: [] }
+        ]
+      )
+
+      SourceMonitor::Security::ParameterSanitizer.sanitize(permitted.to_h)
+    end
+
+    def safe_redirect_path(raw_value)
+      return if raw_value.blank?
+
+      sanitized = SourceMonitor::Security::ParameterSanitizer.sanitize(raw_value.to_s)
+      sanitized.start_with?("/") ? sanitized : nil
+    end
+  end
+end

data/app/helpers/source_monitor/application_helper.rb

@@ -0,0 +1,327 @@
+module SourceMonitor
+  module ApplicationHelper
+    def source_monitor_stylesheet_bundle_tag
+      stylesheet_link_tag("source_monitor/application", "data-turbo-track": "reload")
+    rescue StandardError => error
+      log_source_monitor_asset_error(:stylesheet, error)
+      nil
+    end
+
+    def source_monitor_javascript_bundle_tag
+      javascript_include_tag("source_monitor/application", "data-turbo-track": "reload", type: "module")
+    rescue StandardError => error
+      log_source_monitor_asset_error(:javascript, error)
+      nil
+    end
+
+    def heatmap_bucket_classes(count, max_count)
+      return "bg-slate-100 text-slate-500" if max_count.to_i.zero? || count.to_i.zero?
+
+      ratio = count.to_f / max_count
+
+      case ratio
+      when 0...0.25
+        "bg-blue-100 text-blue-800"
+      when 0.25...0.5
+        "bg-blue-200 text-blue-900"
+      when 0.5...0.75
+        "bg-blue-400 text-white"
+      else
+        "bg-blue-600 text-white"
+      end
+    end
+
+    def fetch_interval_bucket_path(bucket, search_params, selected: false)
+      query = fetch_interval_bucket_query(bucket, search_params, selected: selected)
+      route_helpers = SourceMonitor::Engine.routes.url_helpers
+
+      query.empty? ? route_helpers.sources_path : route_helpers.sources_path(q: query)
+    end
+
+    def fetch_interval_bucket_query(bucket, search_params, selected: false)
+      base = (search_params || {}).dup
+      base = base.except("fetch_interval_minutes_gteq", "fetch_interval_minutes_lt", "fetch_interval_minutes_lteq")
+
+      query = if selected
+        base
+      else
+        updated = base.dup
+        updated["fetch_interval_minutes_gteq"] = bucket.min.to_i.to_s if bucket.respond_to?(:min) && bucket.min
+
+        if bucket.respond_to?(:max) && bucket.max
+          updated["fetch_interval_minutes_lt"] = bucket.max.to_i.to_s
+        else
+          updated.delete("fetch_interval_minutes_lt")
+          updated.delete("fetch_interval_minutes_lteq")
+        end
+
+        updated
+      end
+
+      if query.respond_to?(:compact_blank)
+        query.compact_blank
+      else
+        query.reject { |_key, value| value.respond_to?(:blank?) ? value.blank? : value.nil? }
+      end
+    end
+
+    def fetch_interval_filter_label(bucket, filter)
+      return bucket.label if bucket&.respond_to?(:label)
+      return unless filter
+
+      min = filter[:min]
+      max = filter[:max]
+
+      if min && max
+        "#{min}-#{max} min"
+      elsif min
+        "#{min}+ min"
+      else
+        "Any interval"
+      end
+    end
+
+    def fetch_schedule_window_label(group)
+      start_time = group.respond_to?(:window_start) ? group.window_start : nil
+      end_time = group.respond_to?(:window_end) ? group.window_end : nil
+
+      return unless start_time || end_time
+
+      if start_time && end_time
+        "#{format_schedule_time(start_time)} – #{format_schedule_time(end_time)}"
+      elsif start_time
+        "After #{format_schedule_time(start_time)}"
+      else
+        nil
+      end
+    end
+
+    def format_schedule_time(time)
+      return unless time
+
+      l(time.in_time_zone, format: :short)
+    end
+
+    def human_fetch_interval(minutes)
+      return "—" if minutes.blank?
+
+      total_minutes = minutes.to_i
+      hours, remaining = total_minutes.divmod(60)
+      parts = []
+      parts << "#{hours}h" if hours.positive?
+      parts << "#{remaining}m" if remaining.positive? || parts.empty?
+      parts.join(" ")
+    end
+
+    # Unified status badge helper for both fetch and scrape operations
+    ITEM_SCRAPE_STATUS_LABELS = {
+      "pending" => "Pending",
+      "processing" => "Processing",
+      "success" => "Scraped",
+      "failed" => "Failed",
+      "partial" => "Partial",
+      "disabled" => "Disabled",
+      "idle" => "Not scraped"
+    }.freeze
+
+    # Maps asynchronous workflow states to badge styling/labels shared across the
+    # engine. Item scraping builds on these core states, reusing the same colors
+    # so the UI stays consistent across sources, items, and job dashboards.
+    def async_status_badge(status, show_spinner: true)
+      status_str = status.to_s
+
+      label, classes, spinner = case status_str
+      when "queued"
+        [ "Queued", "bg-amber-100 text-amber-700", show_spinner ]
+      when "pending"
+        [ "Pending", "bg-amber-100 text-amber-700", show_spinner ]
+      when "fetching", "processing"
+        [ "Processing", "bg-blue-100 text-blue-700", show_spinner ]
+      when "success"
+        [ "Completed", "bg-green-100 text-green-700", false ]
+      when "failed"
+        [ "Failed", "bg-rose-100 text-rose-700", false ]
+      when "partial"
+        [ "Partial", "bg-amber-100 text-amber-700", false ]
+      when "disabled"
+        [ "Disabled", "bg-slate-200 text-slate-600", false ]
+      when "idle"
+        [ "Idle", "bg-slate-100 text-slate-600", false ]
+      else
+        [ "Ready", "bg-slate-100 text-slate-600", false ]
+      end
+
+      { label: label, classes: classes, show_spinner: spinner }
+    end
+
+    # Returns a normalized badge payload for the source show/item pages. The
+    # status derives from the item's recorded scrape_status, falls back to the
+    # source configuration, and always lands inside the known status set:
+    # pending, processing, success, failed, partial, disabled, or idle.
+    def item_scrape_status_badge(item:, source: nil, show_spinner: true)
+      status = derive_item_scrape_status(item:, source: source)
+      base_badge = async_status_badge(status, show_spinner: show_spinner)
+      label = ITEM_SCRAPE_STATUS_LABELS.fetch(status) { base_badge[:label] }
+      spinner = base_badge[:show_spinner] && %w[pending processing].include?(status)
+
+      {
+        status: status,
+        label: label,
+        classes: base_badge[:classes],
+        show_spinner: spinner
+      }
+    end
+
+    # Legacy helper for backwards compatibility
+    def fetch_status_badge_classes(status)
+      async_status_badge(status)
+    end
+
+    # Helper to render the loading spinner SVG
+    def loading_spinner_svg(css_class: "mr-1 h-4 w-4 animate-spin text-blue-500")
+      tag.svg(
+        class: css_class,
+        xmlns: "http://www.w3.org/2000/svg",
+        fill: "none",
+        viewBox: "0 0 24 24",
+        aria: { hidden: "true" }
+      ) do
+        concat tag.circle(class: "opacity-25", cx: "12", cy: "12", r: "10", stroke: "currentColor", stroke_width: "4")
+        concat tag.path(class: "opacity-75", fill: "currentColor", d: "M4 12a8 8 0 0 1 8-8v4a4 4 0 0 0-4 4H4z")
+      end
+    end
+
+    def source_health_badge(source, override: nil)
+      return override if override.present?
+
+      status = source&.health_status.presence || "healthy"
+
+      mapping = {
+        "healthy" => { label: "Healthy", classes: "bg-green-100 text-green-700", show_spinner: false },
+        "warning" => { label: "Needs Attention", classes: "bg-amber-100 text-amber-700", show_spinner: false },
+        "critical" => { label: "Failing", classes: "bg-rose-100 text-rose-700", show_spinner: false },
+        "declining" => { label: "Declining", classes: "bg-orange-100 text-orange-700", show_spinner: false },
+        "improving" => { label: "Improving", classes: "bg-sky-100 text-sky-700", show_spinner: false },
+        "auto_paused" => { label: "Auto-Paused", classes: "bg-amber-100 text-amber-700", show_spinner: false },
+        "unknown" => { label: "Unknown", classes: "bg-slate-100 text-slate-600", show_spinner: false }
+      }
+
+      mapping.fetch(status) { mapping.fetch("unknown") }.merge(status: status)
+    end
+
+    def source_health_actions(source)
+      status = source&.health_status.presence || "healthy"
+      helpers = SourceMonitor::Engine.routes.url_helpers
+
+      case status
+      when "critical", "declining"
+        [
+          {
+            key: :full_fetch,
+            label: "Queue Full Fetch",
+            description: "Runs the full fetch pipeline immediately and updates items if the feed responds.",
+            path: helpers.source_fetch_path(source),
+            method: :post,
+            data: { testid: "source-health-action-full_fetch" }
+          },
+          {
+            key: :health_check,
+            label: "Run Health Check",
+            description: "Sends a single request to confirm the feed is reachable without modifying stored items.",
+            path: helpers.source_health_check_path(source),
+            method: :post,
+            data: { testid: "source-health-action-health_check" }
+          }
+        ]
+      when "auto_paused"
+        [
+          {
+            key: :reset,
+            label: "Reset to Active Status",
+            description: "Clears the pause window, failure counters, and schedules the next fetch using the configured interval.",
+            path: helpers.source_health_reset_path(source),
+            method: :post,
+            data: { testid: "source-health-action-reset" }
+          }
+        ]
+      else
+        []
+      end
+    end
+
+    def interactive_health_status?(source, override: nil)
+      return false if override.present?
+
+      %w[critical declining auto_paused].include?(source&.health_status.presence)
+    end
+
+    def table_sort_direction(search_object, attribute)
+      return unless search_object.respond_to?(:sorts)
+
+      sort = search_object.sorts.detect { |s| s && s.name == attribute.to_s }
+      sort&.dir
+    end
+
+    def table_sort_arrow(search_object, attribute, default: nil)
+      direction = table_sort_direction(search_object, attribute) || default&.to_s
+
+      case direction
+      when "asc"
+        "▲"
+      when "desc"
+        "▼"
+      else
+        "↕"
+      end
+    end
+
+    def table_sort_aria(search_object, attribute)
+      direction = table_sort_direction(search_object, attribute)
+
+      case direction
+      when "asc"
+        "ascending"
+      when "desc"
+        "descending"
+      else
+        "none"
+      end
+    end
+
+    def table_sort_link(search_object, attribute, label, frame:, default_order:, secondary: [], html_options: {})
+      sort_targets = [ attribute, *Array(secondary) ]
+      options = {
+        default_order: default_order,
+        hide_indicator: true
+      }.merge(html_options)
+
+      options[:data] = (options[:data] || {}).merge(turbo_frame: frame)
+      options[:data][:turbo_action] ||= "advance"
+
+      sort_link(search_object, attribute, sort_targets, options) do
+        tag.span(label, class: "inline-flex items-center gap-1")
+      end
+    end
+
+    private
+
+    def derive_item_scrape_status(item:, source: nil)
+      return "idle" unless item
+
+      status = item.scrape_status.to_s.presence
+      return status if status.present?
+
+      source ||= item.source
+      return "disabled" if source&.scraping_enabled? == false
+      return "success" if item.scraped_at.present?
+
+      "idle"
+    end
+
+    def log_source_monitor_asset_error(kind, error)
+      return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+      Rails.logger.debug("[SourceMonitor] Skipping #{kind} bundle include: #{error.message}")
+    end
+  end
+end

data/app/jobs/source_monitor/application_job.rb

@@ -0,0 +1,13 @@
+module SourceMonitor
+  parent_job = defined?(::ApplicationJob) ? ::ApplicationJob : ActiveJob::Base
+
+  class ApplicationJob < parent_job
+    class << self
+      # Specify a queue name using SourceMonitor's configuration, ensuring
+      # we respect host application prefixes and overrides.
+      def source_monitor_queue(role)
+        queue_as SourceMonitor.queue_name(role)
+      end
+    end
+  end
+end