source_monitor 0.1.1

data/lib/source_monitor/scraping/item_scraper.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "source_monitor/scraping/item_scraper/adapter_resolver"
+require "source_monitor/scraping/item_scraper/persistence"
+
+module SourceMonitor
+  module Scraping
+    # Orchestrates execution of the configured scraper adapter for an item,
+    # updating the item record and recording a ScrapeLog entry detailing the
+    # outcome. The service is intentionally small so future adapters or
+    # scheduling mechanisms can reuse it for both manual and automated flows.
+    class ItemScraper
+      UnknownAdapterError = Class.new(StandardError)
+
+      Result = Struct.new(:status, :item, :log, :message, :error, keyword_init: true) do
+        def success?
+          status.to_s != "failed"
+        end
+
+        def failed?
+          !success?
+        end
+      end
+
+      attr_reader :item, :source, :adapter_name, :settings, :http, :adapter_resolver, :persistence
+
+      def initialize(item:, source: nil, adapter_name: nil, settings: nil, http: SourceMonitor::HTTP)
+        @item = item
+        @source = source || item&.source
+        @adapter_name = (adapter_name || @source&.scraper_adapter).to_s
+        @settings = settings
+        @http = http
+        @adapter_resolver = SourceMonitor::Scraping::ItemScraper::AdapterResolver.new(name: @adapter_name, source: @source)
+        @persistence = SourceMonitor::Scraping::ItemScraper::Persistence.new(item: @item, source: @source, adapter_name: @adapter_name)
+      end
+
+      def call
+        started_at = Time.current
+        log("scraper:start", started_at:, item:, source:)
+        raise ArgumentError, "Item does not belong to a source" unless source
+        adapter = adapter_resolver.resolve!
+        adapter_result = adapter.call(item:, source:, settings:, http:)
+
+        result = persistence.persist_success(adapter_result:, started_at:)
+        finalize_result(result)
+      rescue UnknownAdapterError => error
+        log("scraper:unknown_adapter", error: error.message)
+        result = persistence.persist_failure(error:, started_at:, message_override: error.message)
+        finalize_result(result)
+      rescue StandardError => error
+        log("scraper:exception", error: error.message)
+        result = persistence.persist_failure(error:, started_at:)
+        finalize_result(result)
+      end
+
+      private
+
+      def finalize_result(result)
+        log(
+          "scraper:finalize",
+          status: result&.status,
+          scrape_status: result&.item&.scrape_status,
+          log_id: result&.log&.id
+        )
+        SourceMonitor::Events.after_item_scraped(result)
+        result
+      end
+
+      def log(stage, **extra)
+        return unless defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
+
+        payload = {
+          stage: "SourceMonitor::Scraping::ItemScraper##{stage}",
+          item_id: item&.id,
+          source_id: source&.id,
+          adapter: adapter_name
+        }.merge(extra.compact)
+        Rails.logger.info("[SourceMonitor::ManualScrape] #{payload.to_json}")
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/source_monitor/scraping/scheduler.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Scraping
+    # Identifies items that still need scraping and enqueue jobs for sources
+    # configured for automatic scraping. This mirrors the feed fetch scheduler
+    # so recurring tasks can keep the scrape queue warm.
+    class Scheduler
+      DEFAULT_BATCH_SIZE = 100
+
+      def self.run(limit: DEFAULT_BATCH_SIZE)
+        new(limit:).run
+      end
+
+      def initialize(limit:)
+        @limit = limit
+      end
+
+      def run
+        items = due_items.limit(limit).includes(:source).to_a
+        return 0 if items.empty?
+
+        items.sum do |item|
+          result = SourceMonitor::Scraping::Enqueuer.enqueue(item: item, source: item.source, reason: :auto)
+          result.enqueued? ? 1 : 0
+        end
+      end
+
+      private
+
+      attr_reader :limit
+
+      def due_items
+        SourceMonitor::Item
+          .joins(:source)
+          .merge(SourceMonitor::Source.active.where(scraping_enabled: true, auto_scrape: true))
+          .where(scraped_at: nil)
+          .where(scrape_status: [ nil, "" ])
+          .order(Arel.sql("sourcemon_items.created_at ASC"))
+      end
+    end
+  end
+end

data/lib/source_monitor/scraping/state.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Scraping
+    # Centralizes scrape status transitions so jobs, schedulers, and UI helpers
+    # keep item states consistent and broadcast changes in one place.
+    module State
+      extend self
+
+      IN_FLIGHT_STATUSES = %w[pending processing].freeze
+
+      def mark_pending!(item, broadcast: false, lock: true)
+        update_status(item, "pending", broadcast:, lock:)
+      end
+
+      def mark_processing!(item, broadcast: true, lock: true)
+        update_status(item, "processing", broadcast:, lock:)
+      end
+
+      def mark_failed!(item, broadcast: true, lock: true, failed_at: Time.current)
+        update_status(
+          item,
+          "failed",
+          broadcast:,
+          lock:,
+          extra: { scraped_at: failed_at || Time.current }
+        )
+      end
+
+      def clear_inflight!(item, broadcast: true, lock: true)
+        with_item(item, lock:) do |record|
+          next unless in_flight?(record.scrape_status)
+
+          record.update_columns(scrape_status: nil)
+          record.assign_attributes(scrape_status: nil)
+        end
+
+        broadcast_item(item) if broadcast
+      end
+
+      def in_flight?(status)
+        IN_FLIGHT_STATUSES.include?(status.to_s)
+      end
+
+      private
+
+      def update_status(item, status, broadcast:, lock:, extra: {})
+        with_item(item, lock:) do |record|
+          attributes = { scrape_status: status }.merge(extra.compact)
+          record.update_columns(attributes)
+          record.assign_attributes(attributes)
+        end
+
+        broadcast_item(item) if broadcast
+      end
+
+      def with_item(item, lock:)
+        return unless item
+
+        if lock
+          item.with_lock do
+            item.reload
+            yield(item)
+          end
+        else
+          yield(item)
+        end
+      rescue ActiveRecord::RecordNotFound
+        nil
+      end
+
+      def broadcast_item(item)
+        SourceMonitor::Realtime.broadcast_item(item)
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/source_monitor/security/authentication.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Security
+    module Authentication
+      def self.authenticate!(controller)
+        call_handler(SourceMonitor.config.authentication.authenticate_handler, controller)
+      end
+
+      def self.authorize!(controller)
+        call_handler(SourceMonitor.config.authentication.authorize_handler, controller)
+      end
+
+      def self.current_user(controller)
+        method_name = preferred_current_user_method(controller)
+        safe_public_send(controller, method_name)
+      end
+
+      def self.user_signed_in?(controller)
+        method_name = preferred_user_signed_in_method(controller)
+
+        if method_name
+          safe_public_send(controller, method_name)
+        else
+          !!current_user(controller)
+        end
+      end
+
+      def self.authentication_configured?
+        config = SourceMonitor.config.authentication
+        config.authenticate_handler.present? || config.authorize_handler.present?
+      end
+
+      def self.authorize_configured?
+        SourceMonitor.config.authentication.authorize_handler.present?
+      end
+
+      def self.authenticate_configured?
+        SourceMonitor.config.authentication.authenticate_handler.present?
+      end
+
+      def self.call_handler(handler, controller)
+        return unless handler
+
+        handler.call(controller)
+      end
+
+      def self.safe_public_send(controller, method_name)
+        return unless method_name
+        return unless controller.respond_to?(method_name, true)
+
+        controller.public_send(method_name)
+      end
+
+      def self.preferred_current_user_method(controller)
+        config = SourceMonitor.config.authentication
+        method_name = config.current_user_method
+        method_name = method_name.to_sym if method_name.respond_to?(:to_sym)
+
+        if method_name
+          method_name
+        elsif controller.respond_to?(:current_user, true)
+          :current_user
+        end
+      end
+
+      def self.preferred_user_signed_in_method(controller)
+        config = SourceMonitor.config.authentication
+        method_name = config.user_signed_in_method
+        method_name = method_name.to_sym if method_name.respond_to?(:to_sym)
+
+        if method_name
+          method_name
+        elsif controller.respond_to?(:user_signed_in?, true)
+          :user_signed_in?
+        end
+      end
+
+      private_class_method :call_handler,
+        :safe_public_send,
+        :preferred_current_user_method,
+        :preferred_user_signed_in_method
+    end
+  end
+end

data/lib/source_monitor/security/parameter_sanitizer.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "action_view"
+
+module SourceMonitor
+  module Security
+    module ParameterSanitizer
+      module_function
+
+      def sanitize(value)
+        case value
+        when ActionController::Parameters
+          sanitize(value.to_unsafe_h)
+        when Hash
+          value.each_with_object({}) do |(key, val), memo|
+            memo[key] = sanitize(val)
+          end
+        when Array
+          value.map { |element| sanitize(element) }
+        when String
+          sanitize_string(value)
+        else
+          value
+        end
+      end
+
+      def sanitize_string(value)
+        stripped = value.to_s
+        return stripped if stripped.blank?
+
+        sanitized = full_sanitizer.sanitize(stripped)
+        sanitized.strip
+      end
+      private_class_method :sanitize_string
+
+      def full_sanitizer
+        @full_sanitizer ||= ActionView::Base.full_sanitizer
+      end
+      private_class_method :full_sanitizer
+    end
+  end
+end

data/lib/source_monitor/sources/turbo_stream_presenter.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module Sources
+    # Presenter for building Turbo Stream responses for source-related actions
+    # Consolidates duplicated response building logic from the controller
+    class TurboStreamPresenter
+      include ActionView::RecordIdentifier
+
+      attr_reader :source, :responder
+
+      def initialize(source:, responder:)
+        @source = source
+        @responder = responder
+      end
+
+      # Builds the complete Turbo Stream response for source deletion
+      # Includes: row removal, heatmap update, empty state (if needed), and redirect (if provided)
+      def render_deletion(metrics:, query:, search_params:, redirect_location: nil)
+        responder.remove_row(source)
+        responder.remove("source_monitor_sources_empty_state")
+
+        render_heatmap_update(metrics:, search_params:)
+        render_empty_state_if_needed(query:)
+        responder.redirect(redirect_location, action: "replace") if redirect_location.present?
+
+        self
+      end
+
+      private
+
+      def render_heatmap_update(metrics:, search_params:)
+        responder.replace(
+          "source_monitor_sources_heatmap",
+          partial: "source_monitor/sources/fetch_interval_heatmap",
+          locals: {
+            fetch_interval_distribution: metrics.fetch_interval_distribution,
+            selected_bucket: metrics.selected_fetch_interval_bucket,
+            search_params:
+          }
+        )
+      end
+
+      def render_empty_state_if_needed(query:)
+        return if query.result.exists?
+
+        responder.append(
+          "source_monitor_sources_table_body",
+          partial: "source_monitor/sources/empty_state_row"
+        )
+      end
+    end
+  end
+end

data/lib/source_monitor/turbo_streams/stream_responder.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module SourceMonitor
+  module TurboStreams
+    class StreamResponder
+      Operation = Struct.new(:action, :target, :partial, :locals, keyword_init: true)
+
+      include ActionView::RecordIdentifier
+
+      attr_reader :operations
+
+      def initialize
+        @operations = []
+      end
+
+      def replace(target, partial:, locals: {})
+        operations << Operation.new(action: :replace, target:, partial:, locals:)
+        self
+      end
+
+      def append(target, partial:, locals: {})
+        operations << Operation.new(action: :append, target:, partial:, locals:)
+        self
+      end
+
+      def replace_details(record, partial:, locals: {})
+        replace(dom_id(record, :details), partial:, locals:)
+      end
+
+      def replace_row(record, partial:, locals: {})
+        replace(dom_id(record, :row), partial:, locals:)
+      end
+
+      def remove(target)
+        operations << Operation.new(action: :remove, target:, partial: nil, locals: nil)
+        self
+      end
+
+      def remove_row(record)
+        remove(dom_id(record, :row))
+      end
+
+      def toast(message:, level: :info, title: nil, delay_ms: 5000)
+        append(
+          "source_monitor_notifications",
+          partial: "source_monitor/shared/toast",
+          locals: {
+            message:,
+            level: level || :info,
+            title:,
+            delay_ms: delay_ms || 5000
+          }
+        )
+      end
+
+      def redirect(url, action: "advance")
+        operations << Operation.new(
+          action: :redirect,
+          target: "source_monitor_redirects",
+          partial: nil,
+          locals: { url:, turbo_action: action }
+        )
+        self
+      end
+
+      def render(view_context)
+        operations.map do |operation|
+          if operation.action == :redirect
+            # Custom redirect action - manually build the turbo-stream tag with URL attribute
+            url = operation.locals[:url]
+            turbo_action = operation.locals[:turbo_action]
+            view_context.tag.send(:"turbo-stream",
+              action: "redirect",
+              target: operation.target,
+              url: url,
+              "visit-action": turbo_action
+            )
+          elsif operation.partial
+            view_context.turbo_stream.public_send(
+              operation.action,
+              operation.target,
+              partial: operation.partial,
+              locals: operation.locals || {}
+            )
+          else
+            view_context.turbo_stream.public_send(
+              operation.action,
+              operation.target
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/version.rb

@@ -0,0 +1,3 @@
+module SourceMonitor
+  VERSION = "0.1.1"
+end

data/lib/source_monitor.rb

@@ -0,0 +1,149 @@
+begin
+  require "solid_queue"
+rescue LoadError
+  # Solid Queue is optional if the host app supplies a different Active Job backend.
+end
+
+begin
+  require "solid_cable"
+rescue LoadError
+  # Solid Cable is optional if the host app uses Redis or another Action Cable adapter.
+end
+
+begin
+  require "turbo-rails"
+rescue LoadError
+  # Turbo is optional but recommended for real-time updates.
+end
+
+begin
+  require "ransack"
+rescue LoadError
+  # Ransack powers search forms when available.
+end
+
+require "source_monitor/version"
+require "active_support/core_ext/module/redefine_method"
+
+SourceMonitor.singleton_class.redefine_method(:table_name_prefix) do
+  SourceMonitor::Engine.table_name_prefix
+end
+
+ActiveSupport.on_load(:active_record) do
+  SourceMonitor.singleton_class.redefine_method(:table_name_prefix) do
+    SourceMonitor::Engine.table_name_prefix
+  end
+end
+
+require "source_monitor/engine"
+require "source_monitor/configuration"
+require "source_monitor/model_extensions"
+require "source_monitor/events"
+require "source_monitor/instrumentation"
+require "source_monitor/metrics"
+require "source_monitor/http"
+require "source_monitor/feedjira_extensions"
+require "source_monitor/dashboard/quick_action"
+require "source_monitor/dashboard/recent_activity"
+require "source_monitor/dashboard/recent_activity_presenter"
+require "source_monitor/dashboard/quick_actions_presenter"
+require "source_monitor/dashboard/queries"
+require "source_monitor/dashboard/turbo_broadcaster"
+require "source_monitor/logs/entry_sync"
+require "source_monitor/logs/filter_set"
+require "source_monitor/logs/query"
+require "source_monitor/logs/table_presenter"
+require "source_monitor/realtime"
+require "source_monitor/analytics/source_fetch_interval_distribution"
+require "source_monitor/analytics/source_activity_rates"
+require "source_monitor/analytics/sources_index_metrics"
+require "source_monitor/jobs/cleanup_options"
+require "source_monitor/jobs/visibility"
+require "source_monitor/jobs/solid_queue_metrics"
+require "source_monitor/security/parameter_sanitizer"
+require "source_monitor/security/authentication"
+require "source_monitor/pagination/paginator"
+require "source_monitor/turbo_streams/stream_responder"
+require "source_monitor/scrapers/base"
+require "source_monitor/scrapers/fetchers/http_fetcher"
+require "source_monitor/scrapers/parsers/readability_parser"
+require "source_monitor/scrapers/readability"
+require "source_monitor/scraping/enqueuer"
+require "source_monitor/scraping/bulk_source_scraper"
+require "source_monitor/scraping/state"
+require "source_monitor/scraping/scheduler"
+require "source_monitor/scraping/item_scraper"
+require "source_monitor/fetching/fetch_error"
+require "source_monitor/fetching/feed_fetcher"
+require "source_monitor/items/retention_pruner"
+require "source_monitor/fetching/fetch_runner"
+require "source_monitor/scheduler"
+require "source_monitor/items/item_creator"
+require "source_monitor/health"
+require "source_monitor/assets"
+
+module SourceMonitor
+  class << self
+    def configure
+      yield config
+      SourceMonitor::ModelExtensions.reload!
+    end
+
+    def config
+      @config ||= Configuration.new
+    end
+
+    def events
+      config.events
+    end
+
+    def reset_configuration!
+      @config = Configuration.new
+      SourceMonitor::ModelExtensions.reload!
+      SourceMonitor::Health.setup!
+      SourceMonitor::Realtime.setup!
+      SourceMonitor::Dashboard::TurboBroadcaster.setup!
+    end
+
+    def queue_name(role)
+      config.queue_name_for(role)
+    end
+
+    def queue_concurrency(role)
+      config.concurrency_for(role)
+    end
+
+    def table_name_prefix
+      SourceMonitor::Engine.table_name_prefix
+    end
+
+    def mission_control_enabled?
+      config.mission_control_enabled
+    end
+
+    def mission_control_dashboard_path
+      raw_path = config.mission_control_dashboard_path
+      resolved = resolve_callable(raw_path)
+      return if resolved.blank?
+
+      valid_dashboard_path?(resolved) ? resolved : nil
+    rescue StandardError
+      nil
+    end
+
+    private
+
+    def resolve_callable(value)
+      value.respond_to?(:call) ? value.call : value
+    end
+
+    def valid_dashboard_path?(value)
+      return true if value.to_s.match?(%r{\Ahttps?://})
+
+      Rails.application.routes.recognize_path(value, method: :get)
+      true
+    rescue ActionController::RoutingError
+      false
+    end
+  end
+end

data/lib/tasks/recover_stalled_fetches.rake

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+namespace :source_monitor do
+  namespace :maintenance do
+    desc "Recover sources stuck in the fetching state when Solid Queue workers crash"
+    task recover_stalled_fetches: :environment do
+      result = SourceMonitor::Fetching::StalledFetchReconciler.call
+
+      recovered_count = result.recovered_source_ids.size
+      removed_jobs_count = result.jobs_removed.size
+
+      puts "Recovered #{recovered_count} stalled sources."
+      puts "Removed #{removed_jobs_count} orphaned job#{'s' unless removed_jobs_count == 1}."
+    end
+  end
+end

data/lib/tasks/source_monitor_assets.rake

@@ -0,0 +1,28 @@
+require "source_monitor/assets/bundler"
+
+namespace :source_monitor do
+  namespace :assets do
+    desc "Build SourceMonitor CSS and JS bundles"
+    task build: :environment do
+      SourceMonitor::Assets::Bundler.build!
+    end
+
+    desc "Verify required SourceMonitor asset bundles exist"
+    task verify: :environment do
+      SourceMonitor::Assets::Bundler.verify!
+    end
+  end
+end
+
+namespace :app do
+  namespace :source_monitor do
+    namespace :assets do
+      task build: "source_monitor:assets:build"
+      task verify: "source_monitor:assets:verify"
+    end
+  end
+end
+
+if defined?(Rake::Task) && Rake::Task.task_defined?("test")
+  Rake::Task["test"].enhance([ "source_monitor:assets:verify" ])
+end